# -*- coding: utf-8 -*- from content_spider.baseSpider import baseSpider from content_spider.baseSpider import baseUpdateSpider from content_spider.baseSpider import fixChapterSpider from content_spider.baseSpider import baseUpdateBookStatusSpider import time import json name = 'feiyuyuedu' allowed_domains = ['www.feiyuyuedu.com'] source = 'zy_feiyuyuedu' source_name = 'feiyuyuedu飞鱼阅读' source_id = 32 base_url = 'https://www.feiyuyuedu.com/apis/zhiyu/{}.php' doc = ''' https://www.feiyuyuedu.com/apis/zhiyu/getBookList.php https://www.feiyuyuedu.com/apis/zhiyu/getBookInfo.php?bookid=520 https://www.feiyuyuedu.com/apis/zhiyu/getChapterList.php?bookid=520 https://www.feiyuyuedu.com/apis/zhiyu/getChapterInfo.php?bookid=520&chapterid=136381 https://www.feiyuyuedu.com/apis/zhiyu/getCateList.php ''' category = [{ "channel_id": 1, "channel_name": "男频", "cate_id": 1, "cate_name": "幻想异能", "category_id":"23", "category_name":"玄幻奇幻" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 2, "cate_name": "悬疑灵异", "category_id":"81", "category_name":"灵异恐怖" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 3, "cate_name": "玄幻奇幻", "category_id":"23", "category_name":"玄幻奇幻" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 4, "cate_name": "仙侠修真", "category_id":"23", "category_name":"玄幻奇幻" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 5, "cate_name": "军事热血", "category_id":"51", "category_name":"特种军旅" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 6, "cate_name": "游戏竞技", "category_id":"19", "category_name":"游戏竞技" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 7, "cate_name": "科幻未来", "category_id":"23", "category_name":"玄幻奇幻" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 8, "cate_name": "古代言情", "category_id":"83", "category_name":"穿越重生" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 9, "cate_name": "现代言情", "category_id":"98", "category_name":"婚恋情感" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 10, "cate_name": "灵异惊悚", "category_id":"114", "category_name":"恐怖惊悚" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 11, "cate_name": "仙侠情缘", "category_id":"97", "category_name":"古典仙侠" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 12, "cate_name": "浪漫校园", "category_id":"104", "category_name":"青春校园" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 13, "cate_name": "民国年代", "category_id":"122", "category_name":"清穿民国" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 15, "cate_name": "幻想甜爽", "category_id":"103", "category_name":"青春纯爱" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 16, "cate_name": "都市生活", "category_id":"54", "category_name":"都市爱情" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 17, "cate_name": "出版", "category_id":"107", "category_name":"其他" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 18, "cate_name": "历史生活", "category_id":"14", "category_name":"历史穿越" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 19, "cate_name": "同人作品", "category_id":"125", "category_name":"衍生同人" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 20, "cate_name": "出版", "category_id":"127", "category_name":"其他作品" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 21, "cate_name": "短篇纪实", "category_id":"107", "category_name":"其他" }] def get_category(category_id): for item in category: if int(category_id) == item['cate_id']: return item return category[0] class feiyuyueduProcess(): name = name allowed_domains = allowed_domains source = source source_name = source_name source_id = source_id def get_start_url(self): return base_url.format('getBookList') def bid_list_result(self, response): result = json.loads(response.text) if result is None or result.get('data') is None: return [] result_list = [] for item in result['data']: result_list.append({'id': item['id']}) return result_list def get_book_info_url(self, bid): return base_url.format('getBookInfo') + '?bookid={}'.format(bid) def book_info_result(self, response): result = json.loads(response.text) result = result['data'] category_info = get_category(result['category_id']) return { 'bid': result['id'], 'name': result['booktitle'], 'author': result['author'], 'intro': result['Introduction'], 'cover': result['cover'], 'keyword': result['Labels'], 'status': result['state'], 'category': category_info['category_name'],'category_id':category_info['category_id'], 'channel': result['channelid'] } def get_chapter_list_url(self, bid): return base_url.format('getChapterList') + '?bookid={}'.format(bid) def chapter_list_result(self, response): result = json.loads(response.text) if result is None or result.get('data') is None: return [] result_list = [] i = 0 for chapter_item in result['data']: i = i+1 result_list.append({ 'source_chapter_id': chapter_item['id'], 'name': chapter_item['Title'], 'sequence': chapter_item['Orders'], 'is_vip': chapter_item['isVip'], 'size': chapter_item['chargeLength'], 'recent_update_at': chapter_item['updateTime'] }) return result_list def get_chapter_content_url(self, bid, cid): return base_url.format('getChapterInfo') + '?bookid={}&chapterid={}'.format(bid,cid) def chapter_content_result(self, response): result = json.loads(response.text) if result is None: return {'content': ''} return { 'content': result['data']['content'] } class feiyuyueduSpider(feiyuyueduProcess,baseSpider): name = name custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class feiyuyueduUpdateSpider(feiyuyueduProcess,baseUpdateSpider): name = name + "update" custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class feiyuyueduFixSpider(feiyuyueduProcess,fixChapterSpider): name = name + 'fix' custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class feiyuyueduBookInfoSpider(feiyuyueduProcess,baseUpdateBookStatusSpider): name = name + "bookinfo" custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' }