feiyu

content_spider/simpleSpider.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+import time
+from content_spider.baseSpider import baseSpider
+from content_spider.baseSpider import baseUpdateSpider
+from content_spider.baseSpider import fixChapterSpider
+from content_spider.baseSpider import baseUpdateBookStatusSpider
+
+
+class simpleProcess(object):
+    name = ''
+    allowed_domains = []
+    source = ''
+    source_name = ''
+    source_id = ''
+
+    def get_start_url(self):
+        raise NotImplementedError
+
+    def bid_list_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return []
+        result_list = []
+        for item in result['data']:
+            result_list.append({'id': item['id']})
+        return result_list
+
+    def get_book_info_url(self, bid):
+        raise NotImplementedError
+
+    def book_info_result(self, response):
+        raise NotImplementedError
+
+    def get_chapter_list_url(self, bid):
+        raise NotImplementedError
+
+    def chapter_list_result(self, response):
+        raise NotImplementedError
+
+    def get_chapter_content_url(self, bid, cid):
+        raise NotImplementedError
+
+    def chapter_content_result(self, response):
+        raise NotImplementedError
+
+
+class simpleProcessSpider(simpleProcess, baseSpider):
+    name = simpleProcess.name
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class simpleProcessupdateSpider(simpleProcess, baseUpdateSpider):
+    name = simpleProcess.name + 'update'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class simpleProcessfixSpider(simpleProcess, fixChapterSpider):
+    name =  simpleProcess.name +'fix'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class simpleProcessBookInfoSpider(simpleProcess, baseUpdateBookStatusSpider):
+    name =  simpleProcess.name +'bookstatusinfo'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }

content_spider/spiders/feiyuyuedu/feiyuyuedu.py

@@ -0,0 +1,254 @@
+# -*- coding: utf-8 -*-
+
+from content_spider.simpleSpider import simpleProcess
+from content_spider.simpleSpider import simpleProcessSpider
+from content_spider.simpleSpider import simpleProcessupdateSpider
+from content_spider.simpleSpider import simpleProcessfixSpider
+from content_spider.simpleSpider import simpleProcessBookInfoSpider
+import json
+
+
+name = 'feiyuyuedu'
+allowed_domains = ['www.feiyuyuedu.com']
+source = 'feiyuyuedu'
+source_name = '飞鱼'
+source_id = 21
+base_url = 'https://www.feiyuyuedu.com/apis/zhiyu/{}.php'
+doc = '''
+https://www.feiyuyuedu.com/apis/zhiyu/getBookList.php
+
+https://www.feiyuyuedu.com/apis/zhiyu/getBookInfo.php?bookid=520
+
+https://www.feiyuyuedu.com/apis/zhiyu/getChapterList.php?bookid=520
+
+https://www.feiyuyuedu.com/apis/zhiyu/getChapterInfo.php?bookid=520&chapterid=136381
+https://www.feiyuyuedu.com/apis/zhiyu/getCateList.php
+
+'''
+
+category =  [{
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 1,
+		"cate_name": "幻想异能",
+        "category_id":"23",
+        "category_name":"玄幻奇幻"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 2,
+		"cate_name": "悬疑灵异",
+        "category_id":"81",
+        "category_name":"灵异恐怖"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 3,
+		"cate_name": "玄幻奇幻",
+        "category_id":"23",
+        "category_name":"玄幻奇幻"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 4,
+		"cate_name": "仙侠修真",
+        "category_id":"23",
+        "category_name":"玄幻奇幻"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 5,
+		"cate_name": "军事热血",
+        "category_id":"51",
+        "category_name":"特种军旅"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 6,
+		"cate_name": "游戏竞技",
+        "category_id":"19",
+        "category_name":"游戏竞技"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 7,
+		"cate_name": "科幻未来",
+        "category_id":"23",
+        "category_name":"玄幻奇幻"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 8,
+		"cate_name": "古代言情",
+        "category_id":"83",
+        "category_name":"穿越重生"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 9,
+		"cate_name": "现代言情",
+        "category_id":"98",
+        "category_name":"婚恋情感"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 10,
+		"cate_name": "灵异惊悚",
+        "category_id":"114",
+        "category_name":"恐怖惊悚"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 11,
+		"cate_name": "仙侠情缘",
+        "category_id":"97",
+        "category_name":"古典仙侠"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 12,
+		"cate_name": "浪漫校园",
+        "category_id":"104",
+        "category_name":"青春校园"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 13,
+		"cate_name": "民国年代",
+        "category_id":"122",
+        "category_name":"清穿民国"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 15,
+		"cate_name": "幻想甜爽",
+        "category_id":"103",
+        "category_name":"青春纯爱"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 16,
+		"cate_name": "都市生活",
+        "category_id":"54",
+        "category_name":"都市爱情"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 17,
+		"cate_name": "出版",
+        "category_id":"107",
+        "category_name":"其他"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 18,
+		"cate_name": "历史生活",
+        "category_id":"14",
+        "category_name":"历史穿越"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 19,
+		"cate_name": "同人作品",
+        "category_id":"125",
+        "category_name":"衍生同人"
+	}, {
+		"channel_id": 1,
+		"channel_name": "男频",
+		"cate_id": 20,
+		"cate_name": "出版",
+        "category_id":"127",
+        "category_name":"其他作品"
+	}, {
+		"channel_id": 2,
+		"channel_name": "女频",
+		"cate_id": 21,
+		"cate_name": "短篇纪实",
+        "category_id":"107",
+        "category_name":"其他"
+	}]
+
+
+
+def get_category(category_id):
+    for item in category:
+        if int(category_id) == item['cate_id']:
+            return item
+    return item[0]
+
+
+class feiyuyueduProcess(simpleProcess):
+    name = name
+    allowed_domains = allowed_domains
+    source = source
+    source_name = source_name
+    source_id = source_id
+
+
+    def get_start_url(self):
+       return base_url.format('getBookList')
+
+    def bid_list_result(self, response):
+        return simpleProcess.bid_list_result(response=response)
+
+    def get_book_info_url(self, bid):
+        return base_url.format('getBookInfo') + '?bookid={}'.format(bid)
+
+    def book_info_result(self, response):
+        result = json.loads(response.text)
+        result = result['data']
+        category_info = get_category(result['category'])
+        return {
+            'bid': result['id'], 'name': result['booktitle'], 'author': result['author'],
+            'intro': result['Introduction'], 'cover': result['cover'], 'keyword': result['Labels'],
+            'status':  result['state'],
+            'category': category_info['category_name'],'category_id':category_info['category_id'],
+            'channel': category_info['channelid']
+        }
+
+    def get_chapter_list_url(self, bid):
+        return base_url.format('getChapterList') + '?bookid={}'.format(bid)
+
+    def chapter_list_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return []
+
+        result_list = []
+        i = 0
+        for chapter_item in result['data']:
+            i = i+1
+            result_list.append({
+                'source_chapter_id': chapter_item['id'], 'name': chapter_item['Title'],
+                'sequence': chapter_item['Orders'], 'is_vip': chapter_item['isVip'],
+                'size': chapter_item['chargeLength'], 'recent_update_at': chapter_item['updateTime']
+            })
+        return result_list
+
+    def get_chapter_content_url(self, bid, cid):
+        return base_url.format('getChapterInfo') + '?bookid={}&chapterid={}'.format(bid,cid)
+
+    def chapter_content_result(self, response):
+        result = json.loads(response.text)
+        if result is None:
+            return {'content': ''}
+
+        return {
+            'content': result['data']['content']
+        }
+    
+class feiyuyueduSpider(feiyuyueduProcess,simpleProcessSpider):
+    pass
+
+
+class feiyuyueduUpdateSpider(feiyuyueduProcess,simpleProcessupdateSpider):
+    pass
+
+
+    
+class feiyuyueduFixSpider(feiyuyueduProcess,simpleProcessfixSpider):
+    pass
+
+
+class feiyuyueduBookInfoSpider(feiyuyueduProcess,simpleProcessBookInfoSpider):
+    pass