|
@@ -17,8 +17,8 @@ class zycontentSpider(scrapy.Spider):
|
|
|
|
|
|
def start_requests(self):
|
|
def start_requests(self):
|
|
self.crawler.stats.set_value('bid_list', [])
|
|
self.crawler.stats.set_value('bid_list', [])
|
|
- param = self.bid
|
|
|
|
- cp_name = '' if self.cp_name is None or self.cp_name == 'zycontent' else self.cp_name
|
|
|
|
|
|
+ param = self.bid if hasattr(self,bid) else None
|
|
|
|
+ cp_name = '' if not hasattr(self,cp_name) or self.cp_name is None or self.cp_name == 'zycontent' else self.cp_name
|
|
if param is not None:
|
|
if param is not None:
|
|
bid_list = param.split(',')
|
|
bid_list = param.split(',')
|
|
for bid in bid_list:
|
|
for bid in bid_list:
|
|
@@ -26,6 +26,7 @@ class zycontentSpider(scrapy.Spider):
|
|
yield scrapy.Request(url, callback=self.parse2, meta={"zhiyu_book_id": bid, "i": 0})
|
|
yield scrapy.Request(url, callback=self.parse2, meta={"zhiyu_book_id": bid, "i": 0})
|
|
else:
|
|
else:
|
|
url = self.base_url + '/booklist' + self.query + '&cp_name={}'.format(cp_name)
|
|
url = self.base_url + '/booklist' + self.query + '&cp_name={}'.format(cp_name)
|
|
|
|
+ self.logger.info(url)
|
|
yield scrapy.Request(url, callback=self.parse1, meta={"zhiyu_book_id": bid, "i": 0})
|
|
yield scrapy.Request(url, callback=self.parse1, meta={"zhiyu_book_id": bid, "i": 0})
|
|
|
|
|
|
def parse1(self,response):
|
|
def parse1(self,response):
|