# -*- coding: utf-8 -*- import time from content_spider.baseSpider import baseSpider from content_spider.baseSpider import baseUpdateSpider from content_spider.baseSpider import fixChapterSpider from content_spider.baseSpider import baseUpdateBookStatusSpider class simpleProcess(object): name = '' allowed_domains = [] source = '' source_name = '' source_id = '' def get_start_url(self): raise NotImplementedError def bid_list_result(self, response): result = json.loads(response.text) if result is None or result.get('data') is None: return [] result_list = [] for item in result['data']: result_list.append({'id': item['id']}) return result_list def get_book_info_url(self, bid): raise NotImplementedError def book_info_result(self, response): raise NotImplementedError def get_chapter_list_url(self, bid): raise NotImplementedError def chapter_list_result(self, response): raise NotImplementedError def get_chapter_content_url(self, bid, cid): raise NotImplementedError def chapter_content_result(self, response): raise NotImplementedError class simpleProcessSpider(simpleProcess, baseSpider): name = simpleProcess.name custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class simpleProcessupdateSpider(simpleProcess, baseUpdateSpider): name = simpleProcess.name + 'update' custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class simpleProcessfixSpider(simpleProcess, fixChapterSpider): name = simpleProcess.name +'fix' custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' } class simpleProcessBookInfoSpider(simpleProcess, baseUpdateBookStatusSpider): name = simpleProcess.name +'bookstatusinfo' custom_settings = { 'DOWNLOAD_DELAY': 0.1, 'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log' }