zhiyu_content_spider/content_spider/spiders/wangyou/wangyou.py

# -*- coding: utf-8 -*-

from content_spider.baseSpider import baseSpider
from content_spider.baseSpider import baseUpdateSpider
from content_spider.baseSpider import fixChapterSpider
from content_spider.baseSpider import baseUpdateBookStatusSpider
from content_spider.Util import md5
import time
import json


name = 'wangyou'
allowed_domains = ['api.wangyou2.com']
source = 'zy_wangyou'
source_name = 'wangyou忘忧'
source_id = 31
sid = '100000106'
key = '4MWJrBwwEdsXnGW426e85f5NYc8TwfJN'
base_url = 'http://api.wangyou2.com/apis/api/{}.php?sid=' + sid + '&sign={}'
doc = 'https://help.2cms.top/web/#/13'


category = [{'channel_id': 1, 'cate_id': '1001', 'cate_name': '热血青春', 'sub_cate_id': '1001001', 'category_id': 67, 'category_name': '校园励志'},
 {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001002', 'category_id': 67, 'category_name': '校园励志'},
 {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001003', 'category_id': 30, 'category_name': '校园黑道'},
 {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001004', 'category_id': 94, 'category_name': '青春爱情'},
 {'channel_id': 1, 'cate_id': '1001', 'cate_name': '', 'sub_cate_id': '1001005', 'category_id': 94, 'category_name': '青春爱情'},
 {'channel_id': 1, 'cate_id': '1002', 'cate_name': '现代都市', 'sub_cate_id': '1002001', 'category_id': 54, 'category_name': '都市爱情'},
 {'channel_id': 1, 'cate_id': '1002', 'cate_name': '', 'sub_cate_id': '1002002', 'category_id': 68, 'category_name': '现代修真'},
 {'channel_id': 1, 'cate_id': '1002', 'cate_name': '', 'sub_cate_id': '1002003', 'category_id': 55, 'category_name': '官场沉浮'},
 {'channel_id': 1, 'cate_id': '1002', 'cate_name': '', 'sub_cate_id': '1002004', 'category_id': 54, 'category_name': '都市爱情'},
 {'channel_id': 1, 'cate_id': '1003', 'cate_name': '悬疑灵异', 'sub_cate_id': '1003001', 'category_id': 81, 'category_name': '灵异恐怖'},
 {'channel_id': 1, 'cate_id': '1003', 'cate_name': '', 'sub_cate_id': '1003002', 'category_id': 81, 'category_name': '灵异恐怖'},
 {'channel_id': 1, 'cate_id': '1003', 'cate_name': '', 'sub_cate_id': '1003003', 'category_id': 81, 'category_name': '灵异恐怖'},
 {'channel_id': 1, 'cate_id': '1003', 'cate_name': '', 'sub_cate_id': '1003004', 'category_id': 81, 'category_name': '灵异恐怖'},
 {'channel_id': 1, 'cate_id': '1004', 'cate_name': '军事历史', 'sub_cate_id': '1004001', 'category_id': 14, 'category_name': '历史穿越'},
 {'channel_id': 1, 'cate_id': '1004', 'cate_name': '', 'sub_cate_id': '1004002', 'category_id': 14, 'category_name': '历史穿越'},
 {'channel_id': 1, 'cate_id': '1004', 'cate_name': '', 'sub_cate_id': '1004003', 'category_id': 14, 'category_name': '历史穿越'},
 {'channel_id': 1, 'cate_id': '1004', 'cate_name': '', 'sub_cate_id': '1004004', 'category_id': 14, 'category_name': '历史穿越'},
 {'channel_id': 1, 'cate_id': '1005', 'cate_name': '玄幻奇幻', 'sub_cate_id': '1005001', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1005', 'cate_name': '', 'sub_cate_id': '1005002', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1005', 'cate_name': '', 'sub_cate_id': '1005003', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1005', 'cate_name': '', 'sub_cate_id': '1005004', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1006', 'cate_name': '武侠仙侠', 'sub_cate_id': '1006001', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1006', 'cate_name': '', 'sub_cate_id': '1006002', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1006', 'cate_name': '', 'sub_cate_id': '1006003', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1006', 'cate_name': '', 'sub_cate_id': '1006004', 'category_id': 23, 'category_name': '玄幻奇幻'},
 {'channel_id': 1, 'cate_id': '1007', 'cate_name': '网游竞技', 'sub_cate_id': '1007001', 'category_id': 19, 'category_name': '游戏竞技'},
 {'channel_id': 1, 'cate_id': '1007', 'cate_name': '', 'sub_cate_id': '1007002', 'category_id': 19, 'category_name': '游戏竞技'},
 {'channel_id': 1, 'cate_id': '1007', 'cate_name': '', 'sub_cate_id': '1007003', 'category_id': 19, 'category_name': '游戏竞技'},
 {'channel_id': 1, 'cate_id': '1007', 'cate_name': '', 'sub_cate_id': '1007004', 'category_id': 19, 'category_name': '游戏竞技'},
 {'channel_id': 2, 'cate_id': '1008', 'cate_name': '热血言情', 'sub_cate_id': '1008001', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008002', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008003', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008004', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1008', 'cate_name': '', 'sub_cate_id': '1008005', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1009', 'cate_name': '古代情缘', 'sub_cate_id': '1009001', 'category_id': 120, 'category_name': '宫斗宅斗'},
 {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009002', 'category_id': 120, 'category_name': '宫斗宅斗'},
 {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009003', 'category_id': 123, 'category_name': '女尊王朝'},
 {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009004', 'category_id': 123, 'category_name': '女尊王朝'},
 {'channel_id': 2, 'cate_id': '1009', 'cate_name': '', 'sub_cate_id': '1009005', 'category_id': 123, 'category_name': '女尊王朝'},
 {'channel_id': 2, 'cate_id': '1010', 'cate_name': '现代爱情', 'sub_cate_id': '1010001', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010002', 'category_id': 98, 'category_name': '婚恋情 感'},
 {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010003', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010004', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1010', 'cate_name': '', 'sub_cate_id': '1010005', 'category_id': 98, 'category_name': '婚恋情感'},
 {'channel_id': 2, 'cate_id': '1011', 'cate_name': '幻想世界', 'sub_cate_id': '1011001', 'category_id': 96, 'category_name': '东方玄幻'},
 {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011002', 'category_id': 96, 'category_name': '东方玄幻'},
 {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011003', 'category_id': 96, 'category_name': '东方玄幻'},
 {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011004', 'category_id': 96, 'category_name': '东方玄幻'},
 {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011005', 'category_id': 96, 'category_name': '东方玄幻'},
 {'channel_id': 2, 'cate_id': '1011', 'cate_name': '', 'sub_cate_id': '1011006', 'category_id': 96, 'category_name': '东方玄幻'}]

def get_sign():
    today = time.strftime("%Y%m%d", time.localtime())
    sign_str = today + sid + key
    return md5(sign_str)    

def get_category(sub_cate_id):
    for item in category:
        if str(sub_cate_id) == item['sub_cate_id']:
            return item
    return item[0]


class wangyouProcess(object):
    name = name
    allowed_domains = allowed_domains
    source = source
    source_name = source_name
    source_id = source_id


    def get_start_url(self):
        sign = get_sign()
        return self.base_url.format('BookList',sign)

    def bid_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []
        result_list = []
        for item in result['data']:
            result_list.append({'id': item['id']})
        return result_list

    def get_book_info_url(self, bid):
        sign = get_sign()
        return self.base_url.format('BookInfo',sign) + '&bookid={}'.format(bid)

    def book_info_result(self, response):
        result = json.loads(response.text)
        result = result['data']
        category_info = get_category(result['ctitle']);
        return {
            'bid': result['id'], 'name': result['bookTitle'], 'author': result['author'],
            'intro': result['introduction'], 'cover': result['cover'], 'keyword': result['labels'],
            'status':result['state'], 'category': category_info['category_name'],'category_id':category_info['category_id'],
            'channel': result['channel_id']
        }

    def get_chapter_list_url(self, bid):
        sign = get_sign()
        return self.base_url.format('BookChapters',sign) + '&bookid={}'.format(bid)

    def chapter_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []

        result_list = []
        i = 0
        for chapter_item in result['data']:
            i = i+1
            result_list.append({
                'source_chapter_id': chapter_item['id'], 'name': chapter_item['title'],
                'sequence': chapter_item['orders'], 'is_vip': 1 if chapter_item['isVip'] else 0,
                'size': 0, 'recent_update_at': chapter_item['postTime']
            })
        return result_list

    def get_chapter_content_url(self, bid, cid):
        sign = get_sign()
        return self.base_url.format('BookChapterInfo',sign) + '&bookid={}&chapterid={}'.format(bid, cid)

    def chapter_content_result(self, response):
        result = json.loads(response.text)
        if result is None:
            return {'content': ''}

        return {
            'content': result['data']['content'],
            'size': len(result['data']['content'])
        }

    
class wangyouSpider(wangyouProcess,baseSpider):
    name = name

    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class wangyouUpdateSpider(wangyouProcess,baseUpdateSpider):
    name = name + "update"
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


    
class wangyouFixSpider(wangyouProcess,fixChapterSpider):
    name = name + 'fix'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class wangyouBookInfoSpider(wangyouProcess,baseUpdateBookStatusSpider):
    name = name + "bookinfo"
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }