banquanmao

content_spider/spiders/banquanmao/banquanmao.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+
+import time
+from content_spider.baseSpider import baseSpider
+from content_spider.pipelines import formatcontent
+from content_spider.baseSpider import baseUpdateSpider
+from content_spider.baseSpider import fixChapterSpider
+from content_spider.baseSpider import baseUpdateBookStatusSpider
+import json
+from  content_spider.Util import md5
+
+
+name = 'banquanmao'
+allowed_domains = ['api.banquanmao.com.cn']
+source = 'zy_banquanmao'
+source_name = 'zy_banquanmao版权猫'
+source_id = 38
+base_url = 'http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method={}&channelkey=6916f4f59737e43d43faeecb759b5e34'
+
+doc = '''
+来量_植宇
+书籍列表：
+http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=booklist&channelkey=6916f4f59737e43d43faeecb759b5e34
+书籍详情：
+http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=bookinfo&channelkey=6916f4f59737e43d43faeecb759b5e34&bid=6109
+章节列表：
+http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=chapterlist&channelkey=6916f4f59737e43d43faeecb759b5e34&bid=6109
+章节内容：
+http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=chapter&channelkey=6916f4f59737e43d43faeecb759b5e34&bid=6109&cid=1021506
+'''
+
+
+
+class banquanmao(object):
+    name = name
+    allowed_domains = allowed_domains
+    source = source
+    source_name = source_name
+    source_id = source_id
+
+    def get_start_url(self):
+        return base_url.format('booklist')
+
+    def bid_list_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return []
+        result_list = []
+        for item in result['data']:
+            result_list.append({'id': item['id']})
+        return result_list
+
+    def get_book_info_url(self, bid):
+        return base_url.format('bookinfo') + '&bid={}'.format(bid)
+
+    def book_info_result(self, response):
+        result = json.loads(response.text)
+        result = result['data']
+        return {
+            'bid': result['id'], 'name': result['booktitle'], 'author': result['author'],
+            'intro': result['Introduction'], 'cover': result['cover'], 'keyword': result['Labels'],
+            'status':  result['state'],
+            'category': result['nclass'],'category_id':result['category_id'],
+            'channel': result['channelid']
+        }
+
+    def get_chapter_list_url(self, bid):
+        return base_url.format('chapterlist') + '&bid={}'.format(bid)
+
+    def chapter_list_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return []
+
+        result_list = []
+        for volumeList in result['data']:
+            for chapter_item in volumeList['chapterlist']:
+                result_list.append({
+                    'source_chapter_id': chapter_item['id'], 'name': chapter_item['Title'],
+                    'sequence': chapter_item['Orders'], 'is_vip': 1 if chapter_item['isVip'] else 0,
+                    'size': chapter_item['chargeLength'], 'recent_update_at': chapter_item['updateTime']
+                })
+        return result_list
+
+    def get_chapter_content_url(self, bid, cid):
+        return base_url.format('chapter') + '&bid={}&cid={}'.format(bid, cid)
+
+    def chapter_content_result(self, response):
+        result = json.loads(response.text)
+        if result is None:
+            return {'content': ''}
+
+        return {
+            'content': result['data']['content']
+        }
+
+
+class banquanmaoSpider(banquanmao, baseSpider):
+    name = 'banquanmao'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class banquanmaoupdateSpider(banquanmao, baseUpdateSpider):
+    name = 'banquanmaoupdate'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class banquanmaofixSpider(banquanmao, fixChapterSpider):
+    name = 'banquanmaofix'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class banquanmaoBookInfoSpider(banquanmao, baseUpdateBookStatusSpider):
+    name = 'banquanmaobookstatusinfo'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }