# -*- coding: utf-8 -*- from content_spider.baseSpider import baseSpider from content_spider.baseSpider import baseUpdateSpider from content_spider.baseSpider import fixChapterSpider from content_spider.baseSpider import baseUpdateBookStatusSpider import time import json name = 'futian' allowed_domains = ['www.futianbook.com'] source = 'zy_futian' source_name = 'futian伏天' source_id = 29 base_url = 'http://www.futianbook.com/api/json' apikey = 'f454124dc5ac861170e8087d67226536' category = [ { "channel_id": 1, "channel_name": "男频", "cate_id": 1, "cate_name": "现代都市", "category_id":"54", "category_name":"都市爱情" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 2, "cate_name": "悬疑灵异", "category_id":"81", "category_name":"灵异恐怖" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 3, "cate_name": "玄幻仙侠", "category_id":"21", "category_name":"武侠仙侠" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 4, "cate_name": "历史军事", "category_id":"51", "category_name":"特种军旅" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 5, "cate_name": "体育电竞", "category_id":"19", "category_name":"游戏竞技" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 6, "cate_name": "科幻末日", "category_id":"23", "category_name":"玄幻奇幻" }, { "channel_id": 1, "channel_name": "男频", "cate_id": 7, "cate_name": "同人其他", "category_id":"127", "category_name":"其他" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 8, "cate_name": "现代言情", "category_id":"87", "category_name":"婚恋情感" }, { "channel_id": 2, "channel_name": "女频", "cate_id": 8, "cate_name": "古代言情", "category_id":"83", "category_name":"穿越重生" } ] def get_category(category_name): for item in category: if category_name == item['cate_name']: return item return category[0] class futianProcess(object): name = name allowed_domains = allowed_domains source = source source_name = source_name source_id = source_id def get_start_url(self): return base_url + '/info?apikey=' + apikey def bid_list_result(self, response): result = json.loads(response.text) if result is None or result.get('msg') is None: return [] result_list = [] for item in result['msg']: result_list.append({'id': item['id']}) return result_list def get_book_info_url(self, bid): return base_url + '/book?apikey={}&bookid={}'.format(apikey, bid) def book_info_result(self, response): result = json.loads(response.text) result = result.get('msg') category_info = get_category(result['ctitle']); return { 'bid': result['id'], 'name': result['title'], 'author': result['author'], 'intro': result['description'], 'cover': result['cover'], 'keyword': '', 'status': result['status'], 'category': category_info['category_name'],'category_id':category_info['category_id'], 'channel': category_info['channel_id'] } def get_chapter_list_url(self, bid): return base_url + '/chapterlist?apikey={}&bookid={}'.format(apikey, bid) def chapter_list_result(self, response): result = json.loads(response.text) if result is None or result.get('msg') is None: return [] result_list = [] i = 0 for chapter_item in result['msg']: i = i+1 result_list.append({ 'source_chapter_id': chapter_item['id'], 'name': chapter_item['title'], 'sequence': i, 'is_vip': chapter_item['isvip'], 'size': 0, 'recent_update_at': chapter_item['update_time'] }) return result_list def get_chapter_content_url(self, bid, cid): return base_url + '/chapter?apikey={}&bookid={}&chapterid={}'.format(apikey, bid, cid) def chapter_content_result(self, response): result = json.loads(response.text) if result is None or result.get('msg') is None: return {'content': ''} return { 'content': result['msg']['content'], 'size': result['msg']['char'], 'sequence': result['msg']['sort'] } class futianSpider(futianProcess,baseSpider): name = name custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class futianUpdateSpider(futianProcess,baseUpdateSpider): name = name + "update" custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class futianFixSpider(futianProcess,fixChapterSpider): name = name + 'fix' custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class futianBookInfoSpider(futianProcess,baseUpdateBookStatusSpider): name = name + "bookinfo" custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' }