zhiyu_content_spider/content_spider/simpleSpider.py

# -*- coding: utf-8 -*-

import time
from content_spider.baseSpider import baseSpider
from content_spider.baseSpider import baseUpdateSpider
from content_spider.baseSpider import fixChapterSpider
from content_spider.baseSpider import baseUpdateBookStatusSpider


class simpleProcess(object):
    name = ''
    allowed_domains = []
    source = ''
    source_name = ''
    source_id = ''

    def get_start_url(self):
        raise NotImplementedError

    def bid_list_result(self, response):
        result = json.loads(response.text)
        if result is None or result.get('data') is None:
            return []
        result_list = []
        for item in result['data']:
            result_list.append({'id': item['id']})
        return result_list

    def get_book_info_url(self, bid):
        raise NotImplementedError

    def book_info_result(self, response):
        raise NotImplementedError

    def get_chapter_list_url(self, bid):
        raise NotImplementedError

    def chapter_list_result(self, response):
        raise NotImplementedError

    def get_chapter_content_url(self, bid, cid):
        raise NotImplementedError

    def chapter_content_result(self, response):
        raise NotImplementedError


class simpleProcessSpider(simpleProcess, baseSpider):
    name = simpleProcess.name
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class simpleProcessupdateSpider(simpleProcess, baseUpdateSpider):
    name = simpleProcess.name + 'update'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class simpleProcessfixSpider(simpleProcess, fixChapterSpider):
    name =  simpleProcess.name +'fix'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }


class simpleProcessBookInfoSpider(simpleProcess, baseUpdateBookStatusSpider):
    name =  simpleProcess.name +'bookstatusinfo'
    custom_settings = {
        'DOWNLOAD_DELAY': 0.1,
        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
    }