zhiyu_content_spider/content_spider/spiders/yuyuedu/yuyuedu.py

# -*- coding: utf-8 -*-

from content_spider.baseSpider import baseSpider
from content_spider.baseSpider import baseUpdateSpider
from content_spider.baseSpider import fixChapterSpider
from content_spider.baseSpider import baseUpdateBookStatusSpider
import time
import json


name = 'yuyuedu'
allowed_domains = ['api.yuread.cn']
source = 'zy_yuyuedu'
source_name = 'yuyuedu娱阅读星河'
source_id = 36
base_url = 'http://api.xingheread.cn/zhiyu/{}'


category =  [
{"sid": "1","title": "男生频道","pid": "0","channel_id":1,'category_name': '武侠仙侠', 'category_id': 21}, 
{"sid": "2","title": "女生频道","pid": "0","channel_id":2,'category_name': '婚恋情感', 'category_id': 98}, 
{"sid": "3","title": "现代都市","pid": "1","channel_id":1,'category_name': '都市爱情', 'category_id': 54}, 
{"sid": "4","title": "玄幻仙侠","pid": "1","channel_id":1,'category_name': '武侠仙侠', 'category_id': 21}, 
{"sid": "5","title": "历史竞技","pid": "1","channel_id":1,'category_name': '历史穿越', 'category_id': 14}, 
{"sid": "6","title": "悬疑灵异","pid": "1","channel_id":1,'category_name': '灵异恐怖', 'category_id': 81}, 
{"sid": "7","title": "科幻末世","pid": "1","channel_id":1,'category_name': '玄幻奇幻', 'category_id': 23}, 
{"sid": "8","title": "现代言情","pid": "2","channel_id":2,'category_name': '婚恋情感', 'category_id': 98}, 
{"sid": "9","title": "穿越古言","pid": "2","channel_id":2,'category_name': '穿越重生', 'category_id': 83}, 
{"sid": "10","title": "玄幻奇幻","pid": "2","channel_id":2,'category_name': '东方玄幻', 'category_id': 96}, 
{"sid": "11","title": "灵异言情","pid": "2","channel_id":2,'category_name': '鬼夫言情', 'category_id': 115}, 
{"sid": "12","title": "唯美同人","pid": "2","channel_id":2,'category_name': '耽美同人', 'category_id': 106}, 
{"sid": "13","title": "官场商场","pid": "3","channel_id":1,'category_name': '官场沉浮', 'category_id': 55}, 
{"sid": "14","title": "热血青春","pid": "3","channel_id":1,'category_name': '青春爱情', 'category_id': 94}, 
{"sid": "15","title": "异术超能","pid": "3","channel_id":1,'category_name': '现代修真', 'category_id': 68}, 
{"sid": "16","title": "热血都市","pid": "3","channel_id":1,'category_name': '都市爱情', 'category_id': 54}, 
{"sid": "17","title": "社会百态","pid": "3","channel_id":1,'category_name': '其他作品', 'category_id': 127}, 
{"sid": "18","title": "乡村生活","pid": "3","channel_id":1,'category_name': '其他作品', 'category_id': 127}, 
{"sid": "19","title": "西方玄幻","pid": "4","channel_id":1,'category_name': '西方玄幻', 'category_id': 22}, 
{"sid": "20","title": "东方玄幻","pid": "4","channel_id":1,'category_name': '武侠仙侠', 'category_id': 21}, 
{"sid": "21","title": "异世修真","pid": "4","channel_id":1,'category_name': '玄幻奇幻', 'category_id': 23}, 
{"sid": "22","title": "神话魔法","pid": "4","channel_id":1,'category_name': '玄幻奇幻', 'category_id': 23}, 
{"sid": "23","title": "战争幻想","pid": "5","channel_id":1,'category_name': '玄幻奇幻', 'category_id': 23}, 
{"sid": "24","title": "历史传记","pid": "5","channel_id":1,'category_name': '特种军旅', 'category_id': 51}, 
{"sid": "25","title": "架空历史","pid": "5","channel_id":1,'category_name': '历史穿越', 'category_id': 14}, 
{"sid": "26","title": "体育竞技","pid": "5","channel_id":1,'category_name': '游戏竞技', 'category_id': 19}, 
{"sid": "27","title": "虚拟网游","pid": "5","channel_id":1,'category_name': '游戏竞技', 'category_id': 19}, 
{"sid": "28","title": "风水秘术","pid": "6","channel_id":1,'category_name': '灵异恐怖', 'category_id': 81}, 
{"sid": "29","title": "悬疑推理","pid": "6","channel_id":1,'category_name': '灵异恐怖', 'category_id': 81}, 
{"sid": "30","title": "恐怖惊悚","pid": "6","channel_id":1,'category_name': '灵异恐怖', 'category_id': 81}, 
{"sid": "31","title": "末世危机","pid": "7","channel_id":1,'category_name': '西方玄幻', 'category_id': 22}, 
{"sid": "32","title": "位面空间","pid": "7","channel_id":1,'category_name': '西方玄幻', 'category_id': 22}, 
{"sid": "33","title": "星际幻想","pid": "7","channel_id":1,'category_name': '西方玄幻', 'category_id': 22}, 
{"sid": "34","title": "科幻机甲","pid": "7","channel_id":1,'category_name': '西方玄幻', 'category_id': 22}, 
{"sid": "35","title": "婚恋职场","pid": "8","channel_id":2,'category_name': '婚恋情感', 'category_id': 98}, 
{"sid": "36","title": "总裁豪门","pid": "8","channel_id":2,'category_name': '豪门总裁', 'category_id': 88}, 
{"sid": "37","title": "现代重生","pid": "8","channel_id":2,'category_name': '都市异能', 'category_id': 118}, 
{"sid": "38","title": "社会女性","pid": "8","channel_id":2,'category_name': '婚恋情感', 'category_id': 98}, 
{"sid": "39","title": "青春校园","pid": "8","channel_id":2,'category_name': '青春校园', 'category_id': 104}, 
{"sid": "40","title": "技能女强","pid": "9","channel_id":2,'category_name': '都市职场', 'category_id': 116}, 
{"sid": "41","title": "宫斗宅斗","pid": "9","channel_id":2,'category_name': '宫斗宅斗', 'category_id': 120}, 
{"sid": "42","title": "经商种田","pid": "9","channel_id":2,'category_name': '经商种田', 'category_id': 84}, 
{"sid": "43","title": "民国旧影","pid": "9","channel_id":2,'category_name': '清穿民国', 'category_id': 122}, 
{"sid": "44","title": "女尊天下","pid": "9","channel_id":2,'category_name': '女尊王朝', 'category_id': 123}, 
{"sid": "45","title": "魔法幻情","pid": "10","channel_id":2,'category_name': '古典仙侠', 'category_id': 97}, 
{"sid": "46","title": "玄幻女强","pid": "10","channel_id":2,'category_name': '东方玄幻', 'category_id': 96}, 
{"sid": "47","title": "仙侠修真","pid": "10","channel_id":2,'category_name': '东方玄幻', 'category_id': 96}, 
{"sid": "48","title": "科幻末世","pid": "10","channel_id":2,'category_name': '东方玄幻', 'category_id': 96}, 
{"sid": "49","title": "空间网游","pid": "10","channel_id":2,'category_name': '游戏', 'category_id': 119}, 
{"sid": "50","title": "现代灵异","pid": "11","channel_id":2,'category_name': '女生灵异', 'category_id': 95}, 
{"sid": "51","title": "民国灵异","pid": "11","channel_id":2,'category_name': '鬼夫言情', 'category_id': 115}, 
{"sid": "52","title": "古代灵异","pid": "11","channel_id":2,'category_name': '鬼夫言情', 'category_id': 115}, 
{"sid": "53","title": "现代耽美","pid": "12","channel_id":2,'category_name': '耽美同人', 'category_id': 106}, 
{"sid": "54","title": "古代耽美","pid": "12","channel_id":2,'category_name': '耽美同人', 'category_id': 106}, 
{"sid": "55","title": "明星同人","pid": "12","channel_id":2,'category_name': '耽美同人', 'category_id': 106}, 
{"sid": "56","title": "动漫同人","pid": "12","channel_id":2,'category_name': '耽美同人', 'category_id': 106}, 
{"sid": "57","title": "短篇","pid": "3","channel_id":1,'category_name': '轻小说', 'category_id': 126}, 
{"sid": "58","title": "短篇","pid": "8","channel_id":2,'category_name': '其他', 'category_id': 107}
]



def get_category(sort_id):
    for item in category:
        if str(sort_id) == item['sid']:
            return item
    return category[0]


class yuyueduProcess():
    name = name
    allowed_domains = allowed_domains
    source = source
    source_name = source_name
    source_id = source_id


    def get_start_url(self):
        return base_url.format('bookList')

    def bid_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []
        result_list = []
        for item in result['data']:
            result_list.append({'id': item['bookid']})
        return result_list

    def get_book_info_url(self, bid):
        return base_url.format('bookInfo') + '?bookId={}'.format(bid)

    def book_info_result(self, response):
        result = json.loads(response.text)
        result = result['data']
        category_info = get_category(result['sort_id']);
        return {
            'bid': result['bookid'], 'name': result['title'], 'author': result['author'],
            'intro': result['description'], 'cover': result['cover'], 'keyword': result['tag'],
            'status':  result['is_finish'], 'category': category_info['category_name'],'category_id':category_info['category_id'],
            'channel': category_info['channel_id']
        }

    def get_chapter_list_url(self, bid):
        return base_url.format('chapterList') + '?bookId={}'.format(bid)

    def chapter_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []

        result_list = []
        i = 0
        for chapter_item in result['data']:
            i = i+1
            result_list.append({
                'source_chapter_id': chapter_item['id'], 'name': chapter_item['title'],
                'sequence': i, 'is_vip': chapter_item['isvip'],
                'size': 0, 'recent_update_at': chapter_item['update_time']
            })
        return result_list

    def get_chapter_content_url(self, bid, cid):
        return base_url.format('content') + '?bookId={}&chapterId={}'.format(bid, cid)

    def chapter_content_result(self, response):
        result = json.loads(response.text)
        if result is None:
            return {'content': ''}

        return {
            'content': result['data']['content'],
            'size':  len(result['data']['content'])
        }

    
class yuyueduSpider(yuyueduProcess,baseSpider):
    name = name
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class yuyueduUpdateSpider(yuyueduProcess,baseUpdateSpider):
    name = name + "update"
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


    
class yuyueduFixSpider(yuyueduProcess,fixChapterSpider):
    name = name + 'fix'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class yuyueduBookInfoSpider(yuyueduProcess,baseUpdateBookStatusSpider):
    name = name + "bookinfo"
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }