|
|
@@ -9,9 +9,9 @@ class YunduyunSpider(scrapy.Spider):
|
|
|
allowed_domains = ['leyuee.com']
|
|
|
start_urls = ['http://www.leyuee.com/services/zwfx.aspx?method=booklist&token=sefaf23h7face']
|
|
|
def parse(self, response):
|
|
|
- res = response.text
|
|
|
+ res = response.text
|
|
|
res = self.json_encode(res)
|
|
|
- self.logger.info(res);
|
|
|
+ self.logger.info(res)
|
|
|
i = 0
|
|
|
mysql = msyqlHelper()
|
|
|
#id = 3
|
|
|
@@ -33,15 +33,15 @@ class YunduyunSpider(scrapy.Spider):
|
|
|
data = dict()
|
|
|
data['ly_bid'] = res['data']['book_id']
|
|
|
data['name'] = res['data']['book_name']
|
|
|
- data['author'] = res['data']['book_author']
|
|
|
- data['intro'] = res['data']['introduction']
|
|
|
- data['cover'] = res['data']['cover_url']
|
|
|
- data['category_name'] = res['data']['book_tags']
|
|
|
- data['category_id'] = res['data']['book_category_id']
|
|
|
- data['status'] = res['data']['book_state']
|
|
|
- data['sequence'] = response.meta['i']
|
|
|
- bid = mysql.insertbook(data)
|
|
|
- mysql.close()
|
|
|
+ data['author'] = res['data']['book_author']
|
|
|
+ data['intro'] = res['data']['introduction']
|
|
|
+ data['cover'] = res['data']['cover_url']
|
|
|
+ data['category_name'] = res['data']['book_tags']
|
|
|
+ data['category_id'] = res['data']['book_category_id']
|
|
|
+ data['status'] = res['data']['book_state']
|
|
|
+ data['sequence'] = response.meta['i']
|
|
|
+ bid = mysql.insertbook(data)
|
|
|
+ mysql.close()
|
|
|
self.logger.info(data)
|
|
|
yield scrapy.Request("http://www.leyuee.com/services/zwfx.aspx?method=chapterlist&bid=%s&token=sefaf23h7face" % res['data']['book_id'],meta={"bid":bid,"book_id":res['data']['book_id']},callback=self.parse3)
|
|
|
|
zhaoyang