|
|
@@ -22,22 +22,23 @@ class zycontentSpider(scrapy.Spider):
|
|
|
mysql = msyqlHelper()
|
|
|
res = response.text
|
|
|
res = self.json_encode(res)
|
|
|
- data = dict()
|
|
|
- data['zhiyu_book_id'] = res['data']['bid']
|
|
|
- data['source_name'] = 'zy_content'
|
|
|
- data['name'] = res['data']['book_name']
|
|
|
- data['author'] = res['data']['author']
|
|
|
- data['intro'] = res['data']['Introduction']
|
|
|
- data['cover'] = res['data']['cover']
|
|
|
- data['category_name'] = res['data']['category_name']
|
|
|
- data['category_id'] = res['data']['category_id']
|
|
|
- data['status'] = res['data']['status']
|
|
|
- data['sequence'] = response.meta['i']
|
|
|
- bid = mysql.insertZyBook(data)
|
|
|
- self.crawler.stats.get_value('bid_list').append(bid)
|
|
|
- mysql.close()
|
|
|
- url = self.base_url + '/chapterlist/{}'.format(res['data']['bid']) + self.query
|
|
|
- yield scrapy.Request(url, meta={"bid": bid, "book_id": res['data']['bid']}, callback=self.parse3)
|
|
|
+ if res['code'] == 10000:
|
|
|
+ data = dict()
|
|
|
+ data['zhiyu_book_id'] = res['data']['bid']
|
|
|
+ data['source_name'] = 'zy_content'
|
|
|
+ data['name'] = res['data']['book_name']
|
|
|
+ data['author'] = res['data']['author']
|
|
|
+ data['intro'] = res['data']['Introduction']
|
|
|
+ data['cover'] = res['data']['cover']
|
|
|
+ data['category_name'] = res['data']['category_name']
|
|
|
+ data['category_id'] = res['data']['category_id']
|
|
|
+ data['status'] = res['data']['status']
|
|
|
+ data['sequence'] = response.meta['i']
|
|
|
+ bid = mysql.insertZyBook(data)
|
|
|
+ self.crawler.stats.get_value('bid_list').append(bid)
|
|
|
+ mysql.close()
|
|
|
+ url = self.base_url + '/chapterlist/{}'.format(res['data']['bid']) + self.query
|
|
|
+ yield scrapy.Request(url, meta={"bid": bid, "book_id": res['data']['bid']}, callback=self.parse3)
|
|
|
|
|
|
def parse3(self, response):
|
|
|
res = response.text
|
zhaoyang