zhiyu_content_spider/content_spider/spiders/feiyuyuedu/feiyuyuedu.py

# -*- coding: utf-8 -*-

from content_spider.baseSpider import baseSpider
from content_spider.baseSpider import baseUpdateSpider
from content_spider.baseSpider import fixChapterSpider
from content_spider.baseSpider import baseUpdateBookStatusSpider
import time
import json


name = 'feiyuyuedu'
allowed_domains = ['www.feiyuyuedu.com']
source = 'zy_feiyuyuedu'
source_name = 'feiyuyuedu飞鱼阅读'
source_id = 32
base_url = 'https://www.feiyuyuedu.com/apis/zhiyu/{}.php'
doc = '''
https://www.feiyuyuedu.com/apis/zhiyu/getBookList.php

https://www.feiyuyuedu.com/apis/zhiyu/getBookInfo.php?bookid=520

https://www.feiyuyuedu.com/apis/zhiyu/getChapterList.php?bookid=520

https://www.feiyuyuedu.com/apis/zhiyu/getChapterInfo.php?bookid=520&chapterid=136381
https://www.feiyuyuedu.com/apis/zhiyu/getCateList.php

'''

category =  [{
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 1,
		"cate_name": "幻想异能",
        "category_id":"23",
        "category_name":"玄幻奇幻"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 2,
		"cate_name": "悬疑灵异",
        "category_id":"81",
        "category_name":"灵异恐怖"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 3,
		"cate_name": "玄幻奇幻",
        "category_id":"23",
        "category_name":"玄幻奇幻"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 4,
		"cate_name": "仙侠修真",
        "category_id":"23",
        "category_name":"玄幻奇幻"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 5,
		"cate_name": "军事热血",
        "category_id":"51",
        "category_name":"特种军旅"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 6,
		"cate_name": "游戏竞技",
        "category_id":"19",
        "category_name":"游戏竞技"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 7,
		"cate_name": "科幻未来",
        "category_id":"23",
        "category_name":"玄幻奇幻"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 8,
		"cate_name": "古代言情",
        "category_id":"83",
        "category_name":"穿越重生"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 9,
		"cate_name": "现代言情",
        "category_id":"98",
        "category_name":"婚恋情感"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 10,
		"cate_name": "灵异惊悚",
        "category_id":"114",
        "category_name":"恐怖惊悚"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 11,
		"cate_name": "仙侠情缘",
        "category_id":"97",
        "category_name":"古典仙侠"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 12,
		"cate_name": "浪漫校园",
        "category_id":"104",
        "category_name":"青春校园"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 13,
		"cate_name": "民国年代",
        "category_id":"122",
        "category_name":"清穿民国"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 15,
		"cate_name": "幻想甜爽",
        "category_id":"103",
        "category_name":"青春纯爱"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 16,
		"cate_name": "都市生活",
        "category_id":"54",
        "category_name":"都市爱情"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 17,
		"cate_name": "出版",
        "category_id":"107",
        "category_name":"其他"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 18,
		"cate_name": "历史生活",
        "category_id":"14",
        "category_name":"历史穿越"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 19,
		"cate_name": "同人作品",
        "category_id":"125",
        "category_name":"衍生同人"
	}, {
		"channel_id": 1,
		"channel_name": "男频",
		"cate_id": 20,
		"cate_name": "出版",
        "category_id":"127",
        "category_name":"其他作品"
	}, {
		"channel_id": 2,
		"channel_name": "女频",
		"cate_id": 21,
		"cate_name": "短篇纪实",
        "category_id":"107",
        "category_name":"其他"
	}]



def get_category(category_id):
    for item in category:
        if int(category_id) == item['cate_id']:
            return item
    return category[0]


class feiyuyueduProcess():
    name = name
    allowed_domains = allowed_domains
    source = source
    source_name = source_name
    source_id = source_id


    def get_start_url(self):
       return base_url.format('getBookList')

    def bid_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []
        result_list = []
        for item in result['data']:
            result_list.append({'id': item['id']})
        return result_list

    def get_book_info_url(self, bid):
        return base_url.format('getBookInfo') + '?bookid={}'.format(bid)

    def book_info_result(self, response):
        result = json.loads(response.text)
        result = result['data']
        category_info = get_category(result['category_id'])
        return {
            'bid': result['id'], 'name': result['booktitle'], 'author': result['author'],
            'intro': result['Introduction'], 'cover': result['cover'], 'keyword': result['Labels'],
            'status':  result['state'],
            'category': category_info['category_name'],'category_id':category_info['category_id'],
            'channel': result['channelid']
        }

    def get_chapter_list_url(self, bid):
        return base_url.format('getChapterList') + '?bookid={}'.format(bid)

    def chapter_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []

        result_list = []
        i = 0
        for chapter_item in result['data']:
            i = i+1
            result_list.append({
                'source_chapter_id': chapter_item['id'], 'name': chapter_item['Title'],
                'sequence': chapter_item['Orders'], 'is_vip': chapter_item['isVip'],
                'size': chapter_item['chargeLength'], 'recent_update_at': chapter_item['updateTime']
            })
        return result_list

    def get_chapter_content_url(self, bid, cid):
        return base_url.format('getChapterInfo') + '?bookid={}&chapterid={}'.format(bid,cid)

    def chapter_content_result(self, response):
        result = json.loads(response.text)
        if result is None:
            return {'content': ''}

        return {
            'content': result['data']['content']
        }
    
class feiyuyueduSpider(feiyuyueduProcess,baseSpider):
    name = name

    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class feiyuyueduUpdateSpider(feiyuyueduProcess,baseUpdateSpider):
    name = name + "update"
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


    
class feiyuyueduFixSpider(feiyuyueduProcess,fixChapterSpider):
    name = name + 'fix'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class feiyuyueduBookInfoSpider(feiyuyueduProcess,baseUpdateBookStatusSpider):
    name = name + "bookinfo"
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }