zhiyu_content_spider/content_spider/spiders/banquanmao/banquanmao.py

# -*- coding: utf-8 -*-

import time
from content_spider.baseSpider import baseSpider
from content_spider.pipelines import formatcontent
from content_spider.baseSpider import baseUpdateSpider
from content_spider.baseSpider import fixChapterSpider
from content_spider.baseSpider import baseUpdateBookStatusSpider
import json
from  content_spider.Util import md5


name = 'banquanmao'
allowed_domains = ['api.banquanmao.com.cn']
source = 'zy_banquanmao'
source_name = 'zy_banquanmao版权猫'
source_id = 38
base_url = 'http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method={}&channelkey={}'

doc = '''
大推_植宇
书籍列表：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=booklist&channelkey=ea1a4a2b760f1f385bc43b39abd0c6e0
书籍详情：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=bookinfo&channelkey=ea1a4a2b760f1f385bc43b39abd0c6e0&bid=6109
章节列表：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=chapterlist&channelkey=ea1a4a2b760f1f385bc43b39abd0c6e0&bid=6109
章节内容：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=chapter&channelkey=ea1a4a2b760f1f385bc43b39abd0c6e0&bid=6109&cid=1021506





来量_植宇
书籍列表：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=booklist&channelkey=6916f4f59737e43d43faeecb759b5e34
书籍详情：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=bookinfo&channelkey=6916f4f59737e43d43faeecb759b5e34&bid=6109
章节列表：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=chapterlist&channelkey=6916f4f59737e43d43faeecb759b5e34&bid=6109
章节内容：
http://api.banquanmao.com.cn/channel/custom/zhiyu/BookSource?method=chapter&channelkey=6916f4f59737e43d43faeecb759b5e34&bid=6109&cid=1021506

'''



class banquanmao(object):
    name = name
    allowed_domains = allowed_domains
    source = source
    source_name = source_name
    source_id = source_id
    channelkey = 'ea1a4a2b760f1f385bc43b39abd0c6e0'

    def get_start_url(self):
        return base_url.format('booklist',self.channelkey)

    def bid_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []
        result_list = []
        for item in result['data']:
            result_list.append({'id': item['id']})
        return result_list

    def get_book_info_url(self, bid):
        return base_url.format('bookinfo',self.channelkey) + '&bid={}'.format(bid)

    def book_info_result(self, response):
        result = json.loads(response.text)
        result = result['data']
        return {
            'bid': result['id'], 'name': result['booktitle'], 'author': result['author'],
            'intro': result['Introduction'], 'cover': result['cover'], 'keyword': result['Labels'],
            'status':  result['state'],
            'category': result['nclass'],'category_id':result['category_id'],
            'channel': result['channelid']
        }

    def get_chapter_list_url(self, bid):
        return base_url.format('chapterlist',self.channelkey) + '&bid={}'.format(bid)

    def chapter_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []

        result_list = []
        for chapter_item in result['data']:
            result_list.append({
                'source_chapter_id': chapter_item['id'], 'name': chapter_item['Title'],
                'sequence': chapter_item['Orders'], 'is_vip': 1 if chapter_item['isVip'] else 0,
                'size': chapter_item['chargeLength'], 'recent_update_at': chapter_item['updateTime']
            })
        return result_list

    def get_chapter_content_url(self, bid, cid):
        return base_url.format('chapter',self.channelkey) + '&bid={}&cid={}'.format(bid, cid)

    def chapter_content_result(self, response):
        result = json.loads(response.text)
        if result is None:
            return {'content': ''}

        return {
            'content': result['data']['content']
        }


class banquanmaoSpider(banquanmao, baseSpider):
    name = 'banquanmao'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class banquanmaoupdateSpider(banquanmao, baseUpdateSpider):
    name = 'banquanmaoupdate'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class banquanmaofixSpider(banquanmao, fixChapterSpider):
    name = 'banquanmaofix'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class banquanmaoBookInfoSpider(banquanmao, baseUpdateBookStatusSpider):
    name = 'banquanmaobookstatusinfo'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }






#-------------------------------------------------------------------------------------------------

class lailiangSpider(banquanmao, baseSpider):
    name = 'lailiang'
    source = 'zy_lailiang'
    source_name = 'zy_lailiang来量'
    source_id = 37
    channelkey = '6916f4f59737e43d43faeecb759b5e34'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class lailiangupdateSpider(banquanmao, baseUpdateSpider):
    name = 'lailiangupdate'
    source = 'zy_lailiang'
    source_name = 'zy_lailiang来量'
    source_id = 37
    channelkey = '6916f4f59737e43d43faeecb759b5e34'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class lailiangfixSpider(banquanmao, fixChapterSpider):
    name = 'lailiangfix'
    source = 'zy_lailiang'
    source_name = 'zy_lailiang来量'
    source_id = 37
    channelkey = '6916f4f59737e43d43faeecb759b5e34'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class lailiangBookInfoSpider(banquanmao, baseUpdateBookStatusSpider):
    name = 'lailiangbookstatusinfo'
    source = 'zy_lailiang'
    source_name = 'zy_lailiang来量'
    source_id = 37
    channelkey = '6916f4f59737e43d43faeecb759b5e34'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'SOURCE': source,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }