feiyu

content_spider/simpleSpider.py

@@ -1,78 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import time
-from content_spider.baseSpider import baseSpider
-from content_spider.baseSpider import baseUpdateSpider
-from content_spider.baseSpider import fixChapterSpider
-from content_spider.baseSpider import baseUpdateBookStatusSpider
-import json
-
-
-class simpleProcess(object):
-    name = ''
-    allowed_domains = []
-    source = ''
-    source_name = ''
-    source_id = ''
-
-    def get_start_url(self):
-        raise NotImplementedError
-
-    def bid_list_result(self, response):
-        result = json.loads(response.text)
-        if result is None or result.get('data') is None:
-            return []
-        result_list = []
-        for item in result['data']:
-            result_list.append({'id': item['id']})
-        return result_list
-
-    def get_book_info_url(self, bid):
-        raise NotImplementedError
-
-    def book_info_result(self, response):
-        raise NotImplementedError
-
-    def get_chapter_list_url(self, bid):
-        raise NotImplementedError
-
-    def chapter_list_result(self, response):
-        raise NotImplementedError
-
-    def get_chapter_content_url(self, bid, cid):
-        raise NotImplementedError
-
-    def chapter_content_result(self, response):
-        raise NotImplementedError
-
-
-class simpleProcessSpider(simpleProcess, baseSpider):
-    name = simpleProcess.name
-    custom_settings = {
-        'DOWNLOAD_DELAY': 0.1,
-        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
-    }
-
-
-class simpleProcessupdateSpider(simpleProcess, baseUpdateSpider):
-    name = simpleProcess.name + 'update'
-    custom_settings = {
-        'DOWNLOAD_DELAY': 0.1,
-        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
-    }
-
-
-class simpleProcessfixSpider(simpleProcess, fixChapterSpider):
-    name =  simpleProcess.name +'fix'
-    custom_settings = {
-        'DOWNLOAD_DELAY': 0.1,
-        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
-    }
-
-
-class simpleProcessBookInfoSpider(simpleProcess, baseUpdateBookStatusSpider):
-    name =  simpleProcess.name +'bookstatusinfo'
-    custom_settings = {
-        'DOWNLOAD_DELAY': 0.1,
-        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
-    }

content_spider/spiders/feiyuyuedu/feiyuyuedu.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
-from content_spider.simpleSpider import simpleProcess
-from content_spider.simpleSpider import simpleProcessSpider
-from content_spider.simpleSpider import simpleProcessupdateSpider
-from content_spider.simpleSpider import simpleProcessfixSpider
-from content_spider.simpleSpider import simpleProcessBookInfoSpider
+from content_spider.baseSpider import baseSpider
+from content_spider.baseSpider import baseUpdateSpider
+from content_spider.baseSpider import fixChapterSpider
+from content_spider.baseSpider import baseUpdateBookStatusSpider
+
 import json
 
 
@@ -177,7 +177,7 @@ def get_category(category_id):
     return item[0]
 
 
-class feiyuyueduProcess(simpleProcess):
+class feiyuyueduProcess():
     name = name
     allowed_domains = allowed_domains
     source = source
@@ -203,7 +203,7 @@ class feiyuyueduProcess(simpleProcess):
     def book_info_result(self, response):
         result = json.loads(response.text)
         result = result['data']
-        category_info = get_category(result['category'])
+        category_info = get_category(result['category_id'])
         return {
             'bid': result['id'], 'name': result['booktitle'], 'author': result['author'],
             'intro': result['Introduction'], 'cover': result['cover'], 'keyword': result['Labels'],
@@ -243,7 +243,7 @@ class feiyuyueduProcess(simpleProcess):
             'content': result['data']['content']
         }
     
-class feiyuyueduSpider(feiyuyueduProcess,simpleProcessSpider):
+class feiyuyueduSpider(feiyuyueduProcess,baseSpider):
     name = name
     allowed_domains = allowed_domains
     source = source
@@ -251,7 +251,7 @@ class feiyuyueduSpider(feiyuyueduProcess,simpleProcessSpider):
     source_id = source_id
 
 
-class feiyuyueduUpdateSpider(feiyuyueduProcess,simpleProcessupdateSpider):
+class feiyuyueduUpdateSpider(feiyuyueduProcess,baseUpdateSpider):
     name = name + "update"
     allowed_domains = allowed_domains
     source = source
@@ -260,7 +260,7 @@ class feiyuyueduUpdateSpider(feiyuyueduProcess,simpleProcessupdateSpider):
 
 
     
-class feiyuyueduFixSpider(feiyuyueduProcess,simpleProcessfixSpider):
+class feiyuyueduFixSpider(feiyuyueduProcess,fixChapterSpider):
     name = name + 'fix'
     allowed_domains = allowed_domains
     source = source
@@ -268,7 +268,7 @@ class feiyuyueduFixSpider(feiyuyueduProcess,simpleProcessfixSpider):
     source_id = source_id
 
 
-class feiyuyueduBookInfoSpider(feiyuyueduProcess,simpleProcessBookInfoSpider):
+class feiyuyueduBookInfoSpider(feiyuyueduProcess,baseUpdateBookStatusSpider):
     name = name + "bookinfo"
     allowed_domains = allowed_domains
     source = source