# -*- coding: utf-8 -*- from content_spider.baseSpider import baseSpider from content_spider.baseSpider import baseUpdateSpider from content_spider.baseSpider import fixChapterSpider from content_spider.baseSpider import baseUpdateBookStatusSpider from content_spider.Util import md5 import time import json name = 'wangyou' allowed_domains = ['api.wangyou2.com'] source = 'zy_wangyou' source_name = 'wangyou忘忧' source_id = 31 sid = '100000106' key = '4MWJrBwwEdsXnGW426e85f5NYc8TwfJN' base_url = 'http://api.wangyou2.com/apis/api/{}.php?sid=' + sid + '&sign={}' doc = 'https://help.2cms.top/web/#/13' category = [{'channel_id': 1, 'cate_id': '1001', 'cate_name': '热血青春', 'sub_cate_id': '1001001', 'category_id': 67, 'category_name': '校园励志'}, {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001002', 'category_id': 67, 'category_name': '校园励志'}, {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001003', 'category_id': 30, 'category_name': '校园黑道'}, {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001004', 'category_id': 94, 'category_name': '青春爱情'}, {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001005', 'category_id': 94, 'category_name': '青春爱情'}, {'channel_id': 1, 'cate_id': '1002', 'cate_name': '现代都市', 'sub_cate_id': '1002001', 'category_id': 54, 'category_name': '都市爱情'}, {'channel_id': 1, 'cate_id': '1002', 'cate_name': '', 'sub_cate_id': '1002002', 'category_id': 68, 'category_name': '现代修真'}, {'channel_id': 1, 'cate_id': '1002', 'cate_name': '', 'sub_cate_id': '1002003', 'category_id': 55, 'category_name': '官场沉浮'}, {'channel_id': 1, 'cate_id': '1002', 'cate_name': '', 'sub_cate_id': '1002004', 'category_id': 54, 'category_name': '都市爱情'}, {'channel_id': 1, 'cate_id': '1003', 'cate_name': '悬疑灵异', 'sub_cate_id': '1003001', 'category_id': 81, 'category_name': '灵异恐怖'}, {'channel_id': 1, 'cate_id': '1003', 'cate_name': '', 'sub_cate_id': '1003002', 'category_id': 81, 'category_name': '灵异恐怖'}, {'channel_id': 1, 'cate_id': '1003', 'cate_name': '', 'sub_cate_id': '1003003', 'category_id': 81, 'category_name': '灵异恐怖'}, {'channel_id': 1, 'cate_id': '1003', 'cate_name': '', 'sub_cate_id': '1003004', 'category_id': 81, 'category_name': '灵异恐怖'}, {'channel_id': 1, 'cate_id': '1004', 'cate_name': '军事历史', 'sub_cate_id': '1004001', 'category_id': 14, 'category_name': '历史穿越'}, {'channel_id': 1, 'cate_id': '1004', 'cate_name': '', 'sub_cate_id': '1004002', 'category_id': 14, 'category_name': '历史穿越'}, {'channel_id': 1, 'cate_id': '1004', 'cate_name': '', 'sub_cate_id': '1004003', 'category_id': 14, 'category_name': '历史穿越'}, {'channel_id': 1, 'cate_id': '1004', 'cate_name': '', 'sub_cate_id': '1004004', 'category_id': 14, 'category_name': '历史穿越'}, {'channel_id': 1, 'cate_id': '1005', 'cate_name': '玄幻奇幻', 'sub_cate_id': '1005001', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1005', 'cate_name': '', 'sub_cate_id': '1005002', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1005', 'cate_name': '', 'sub_cate_id': '1005003', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1005', 'cate_name': '', 'sub_cate_id': '1005004', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1006', 'cate_name': '武侠仙侠', 'sub_cate_id': '1006001', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1006', 'cate_name': '', 'sub_cate_id': '1006002', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1006', 'cate_name': '', 'sub_cate_id': '1006003', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1006', 'cate_name': '', 'sub_cate_id': '1006004', 'category_id': 23, 'category_name': '玄幻奇幻'}, {'channel_id': 1, 'cate_id': '1007', 'cate_name': '网游竞技', 'sub_cate_id': '1007001', 'category_id': 19, 'category_name': '游戏竞技'}, {'channel_id': 1, 'cate_id': '1007', 'cate_name': '', 'sub_cate_id': '1007002', 'category_id': 19, 'category_name': '游戏竞技'}, {'channel_id': 1, 'cate_id': '1007', 'cate_name': '', 'sub_cate_id': '1007003', 'category_id': 19, 'category_name': '游戏竞技'}, {'channel_id': 1, 'cate_id': '1007', 'cate_name': '', 'sub_cate_id': '1007004', 'category_id': 19, 'category_name': '游戏竞技'}, {'channel_id': 2, 'cate_id': '1008', 'cate_name': '热血言情', 'sub_cate_id': '1008001', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008002', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008003', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008004', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008005', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1009', 'cate_name': '古代情缘', 'sub_cate_id': '1009001', 'category_id': 120, 'category_name': '宫斗宅斗'}, {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009002', 'category_id': 120, 'category_name': '宫斗宅斗'}, {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009003', 'category_id': 123, 'category_name': '女尊王朝'}, {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009004', 'category_id': 123, 'category_name': '女尊王朝'}, {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009005', 'category_id': 123, 'category_name': '女尊王朝'}, {'channel_id': 2, 'cate_id': '1010', 'cate_name': '现代爱情', 'sub_cate_id': '1010001', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010002', 'category_id': 98, 'category_name': '婚恋情 感'}, {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010003', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010004', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010005', 'category_id': 98, 'category_name': '婚恋情感'}, {'channel_id': 2, 'cate_id': '1011', 'cate_name': '幻想世界', 'sub_cate_id': '1011001', 'category_id': 96, 'category_name': '东方玄幻'}, {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011002', 'category_id': 96, 'category_name': '东方玄幻'}, {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011003', 'category_id': 96, 'category_name': '东方玄幻'}, {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011004', 'category_id': 96, 'category_name': '东方玄幻'}, {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011005', 'category_id': 96, 'category_name': '东方玄幻'}, {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011006', 'category_id': 96, 'category_name': '东方玄幻'}] def get_sign(): today = time.strftime("%Y%m%d", time.localtime()) sign_str = today + sid + key return md5(sign_str) def get_category(sub_cate_id): for item in category: if str(sub_cate_id) == item['sub_cate_id']: return item return item[0] class wangyouProcess(object): name = name allowed_domains = allowed_domains source = source source_name = source_name source_id = source_id def get_start_url(self): sign = get_sign() return base_url.format('BookList',sign) def bid_list_result(self, response): result = json.loads(response.text) if result is None or result.get('data') is None: return [] result_list = [] for item in result['data']: result_list.append({'id': item['id']}) return result_list def get_book_info_url(self, bid): sign = get_sign() return base_url.format('BookInfo',sign) + '&bookid={}'.format(bid) def book_info_result(self, response): result = json.loads(response.text) result = result['data'] category_info = get_category(result['ctitle']); return { 'bid': result['id'], 'name': result['bookTitle'], 'author': result['author'], 'intro': result['introduction'], 'cover': result['cover'], 'keyword': result['labels'], 'status':result['state'], 'category': category_info['category_name'],'category_id':category_info['category_id'], 'channel': result['channel_id'] } def get_chapter_list_url(self, bid): sign = get_sign() return base_url.format('BookChapters',sign) + '&bookid={}'.format(bid) def chapter_list_result(self, response): result = json.loads(response.text) if result is None or result.get('data') is None: return [] result_list = [] i = 0 for chapter_item in result['data']: i = i+1 result_list.append({ 'source_chapter_id': chapter_item['id'], 'name': chapter_item['title'], 'sequence': chapter_item['orders'], 'is_vip': 1 if chapter_item['isVip'] else 0, 'size': 0, 'recent_update_at': chapter_item['postTime'] }) return result_list def get_chapter_content_url(self, bid, cid): sign = get_sign() return base_url.format('BookChapterInfo',sign) + '&bookid={}&chapterid={}'.format(bid, cid) def chapter_content_result(self, response): result = json.loads(response.text) if result is None: return {'content': ''} return { 'content': result['data']['content'], 'size': len(result['data']['content']) } class wangyouSpider(wangyouProcess,baseSpider): name = name custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class wangyouUpdateSpider(wangyouProcess,baseUpdateSpider): name = name + "update" custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class wangyouFixSpider(wangyouProcess,fixChapterSpider): name = name + 'fix' custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class wangyouBookInfoSpider(wangyouProcess,baseUpdateBookStatusSpider): name = name + "bookinfo" custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'SOURCE': source, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' }