aiyouhuyu

content_spider/spiders/aiyouhuyu/aiyouhuyu.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+
+from content_spider.baseSpider import baseSpider
+from content_spider.baseSpider import baseUpdateSpider
+from content_spider.baseSpider import fixChapterSpider
+from content_spider.baseSpider import baseUpdateBookStatusSpider
+import time
+import json
+
+
+name = 'aiyouhuyu'
+allowed_domains = ['bookapi.fensebook.com']
+source = 'zy_aiyouhuyu'
+source_name = 'aiyouhuyu哎呦互娱'
+source_id = 34
+appKey = 'ToZYZSY202304101759'
+appSecret = '9000bd89f6cc8e27619d10883f7f6b51'
+base_url = 'http://bookapi.fensebook.com/bookApi/{}?appKey='+appKey + '&appSecret='+appSecret
+
+
+category =  [{'id': 1001, 'name': '都市', 'channel_id': 1, 'category_name': '都市爱情', 'category_id': 54},
+ {'id': 1002, 'name': '玄幻', 'channel_id': 1, 'category_name': '武侠仙侠', 'category_id': 21},
+ {'id': 1003, 'name': '仙侠', 'channel_id': 1, 'category_name': '武侠仙侠', 'category_id': 21},
+ {'id': 1004, 'name': '历史', 'channel_id': 1, 'category_name': '历史穿越', 'category_id': 14},
+ {'id': 1005, 'name': '灵异', 'channel_id': 1, 'category_name': '灵异恐怖', 'category_id': 81},
+ {'id': 1006, 'name': '游戏', 'channel_id': 1, 'category_name': '游戏竞技', 'category_id': 19},
+ {'id': 1007, 'name': '科幻', 'channel_id': 1, 'category_name': '玄幻奇幻', 'category_id': 23},
+ {'id': 1008, 'name': '奇幻', 'channel_id': 1, 'category_name': '玄幻奇幻', 'category_id': 23},
+ {'id': 1009, 'name': '军事', 'channel_id': 1, 'category_name': '历史穿越', 'category_id': 14},
+ {'id': 1012, 'name': '武侠', 'channel_id': 1, 'category_name': '武侠仙侠', 'category_id': 21},
+ {'id': 1014, 'name': '竞技', 'channel_id': 1, 'category_name': '游戏竞技', 'category_id': 19},
+ {'id': 1016, 'name': '悬疑', 'channel_id': 1, 'category_name': '灵异恐怖', 'category_id': 81},
+ {'id': 1019, 'name': '二次元', 'channel_id': 1, 'category_name': '其他作品', 'category_id': 127},
+ {'id': 1020, 'name': '轻小说', 'channel_id': 1, 'category_name': '其他作品', 'category_id': 127},
+ {'id': 1021, 'name': '现实', 'channel_id': 1, 'category_name': '其他作品', 'category_id': 127},
+ {'id': 2001, 'name': '现代言情', 'channel_id': 2, 'category_name': '婚恋情感', 'category_id': 87},
+ {'id': 2002, 'name': '古代言情', 'channel_id': 2, 'category_name': '穿越重生', 'category_id': 83},
+ {'id': 2003, 'name': '幻想言情', 'channel_id': 2, 'category_name': '东方玄幻', 'category_id': 96},
+ {'id': 2004, 'name': '青春校园', 'channel_id': 2, 'category_name': '青春校园', 'category_id': 104},
+ {'id': 2007, 'name': '短篇', 'channel_id': 2, 'category_name': '其他', 'category_id': 107},
+ {'id': 2009, 'name': '悬疑', 'channel_id': 2, 'category_name': '悬疑探险', 'category_id': 113},
+ {'id': 2011, 'name': '穿越架空', 'channel_id': 2, 'category_name': '穿越重生', 'category_id': 83},
+ {'id': 2012, 'name': '纯爱', 'channel_id': 2, 'category_name': '青春纯爱', 'category_id': 103},
+ {'id': 2013, 'name': '二次元', 'channel_id': 2, 'category_name': '其他', 'category_id': 107},
+ {'id': 2014, 'name': '科幻空间', 'channel_id': 2, 'category_name': '东方玄幻', 'category_id': 96},
+ {'id': 2015, 'name': '玄幻言情', 'channel_id': 2, 'category_name': '东方玄幻', 'category_id': 96},
+ {'id': 2016, 'name': '仙侠奇缘', 'channel_id': 2, 'category_name': '东方玄幻', 'category_id': 96},
+ {'id': 2017, 'name': '浪漫青春', 'channel_id': 2, 'category_name': '青春纯爱', 'category_id': 103},
+ {'id': 2018, 'name': '游戏竞技', 'channel_id': 2, 'category_name': '游戏', 'category_id': 113},
+ {'id': 2019, 'name': '轻小说', 'channel_id': 2, 'category_name': '其他', 'category_id': 107},
+ {'id': 2020, 'name': '现实生活', 'channel_id': 2, 'category_name': '其他', 'category_id': 107},
+ {'id': 2021, 'name': '职场沉浮', 'channel_id': 2, 'category_name': '其他', 'category_id': 107}]
+
+
+
+def get_category(category_name):
+    for item in category:
+        if category_name == item['name']:
+            return item
+    return category[0]
+
+
+class aiyouhuyuProcess():
+    name = name
+    allowed_domains = allowed_domains
+    source = source
+    source_name = source_name
+    source_id = source_id
+
+
+    def get_start_url(self):
+        return base_url.format('getBookList')
+
+    def bid_list_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return []
+        result_list = []
+        for item in result['data']:
+            result_list.append({'id': item['bookId']})
+        return result_list
+
+    def get_book_info_url(self, bid):
+        return base_url.format('getBookInfo') + '&bookId={}'.format(bid)
+
+    def book_info_result(self, response):
+        result = json.loads(response.text)
+        result = result.get('data')
+        category_info = get_category(result['category'])
+        return {
+            'bid': result['bookId'], 'name': result['bookName'], 'author': result['author'],
+            'intro': result['desc'], 'cover': result['cover'],
+            'keyword': '' if result['keyWords'] is None else result['keyWords'],
+            'status':  result['isFinished'],'category': category_info['category_name'],'category_id':category_info['category_id'],
+            'channel': result['channelid']
+        }
+
+    def get_chapter_list_url(self, bid):
+        return base_url.format('getChapterList') + '&bookId={}'.format(bid)
+
+    def chapter_list_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return []
+
+        result_list = []
+        i = 0
+        for chapter_item in result['data']:
+            i = i+1
+            result_list.append({
+                'source_chapter_id': chapter_item['id'], 'name': chapter_item['title'],
+                'sequence': chapter_item['sort'], 'is_vip': 0 if int(chapter_item['vip']) == 1 else 1,
+                'size': chapter_item['words'], 'recent_update_at': ''
+            })
+        return result_list
+
+    def get_chapter_content_url(self, bid, cid):
+        return base_url.format('getContent') + '&bookId={}&chapterId={}'.format(bid, cid)
+
+    def chapter_content_result(self, response):
+        result = json.loads(response.text)
+        if result is None or result.get('data') is None:
+            return {'content': ''}
+
+        return {
+            'content': result['data']['content'],
+            'size': len(result['data']['content'])
+        }
+    
+class aiyouhuyuSpider(aiyouhuyuProcess,baseSpider):
+    name = name
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class aiyouhuyuUpdateSpider(aiyouhuyuProcess,baseUpdateSpider):
+    name = name + "update"
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+    
+class aiyouhuyuFixSpider(aiyouhuyuProcess,fixChapterSpider):
+    name = name + 'fix'
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }
+
+
+class aiyouhuyuBookInfoSpider(aiyouhuyuProcess,baseUpdateBookStatusSpider):
+    name = name + "bookinfo"
+    custom_settings = {
+        'DOWNLOAD_DELAY': 0.1,
+        'SOURCE': source,
+        'LOG_FILE': 'content_spider/log/' + name + time.strftime("%Y-%m-%d", time.localtime()) + '.log'
+    }

content_spider/spiders/feiyuyuedu/feiyuyuedu.py

@@ -174,7 +174,7 @@ def get_category(category_id):
     for item in category:
         if int(category_id) == item['cate_id']:
             return item
-    return item[0]
+    return category[0]
 
 
 class feiyuyueduProcess():

content_spider/spiders/futian/futian.py

@@ -97,7 +97,7 @@ def get_category(category_name):
     for item in category:
         if category_name == item['cate_name']:
             return item
-    return item[0]
+    return category[0]
 
 
 class futianProcess(object):

content_spider/spiders/judian/judian.py

@@ -54,7 +54,7 @@ def get_category(type):
     for item in category:
         if int(type) == item['type']:
             return item
-    return item[0]
+    return category[0]
 
 
 class judianProcess(object):

content_spider/spiders/wangyou/wangyou.py

@@ -80,7 +80,7 @@ def get_category(sub_cate_id):
     for item in category:
         if str(sub_cate_id) == item['sub_cate_id']:
             return item
-    return item[0]
+    return category[0]
 
 
 class wangyouProcess(object):