Scrapy：如何将 url_id 与抓取的数据一起存储

Question

from scrapy import Spider, Requestfrom selenium import webdriverclass MySpider(Spider):    name = "my_spider"    def __init__(self):        self.browser = webdriver.Chrome(executable_path='E:/chromedriver')        self.browser.set_page_load_timeout(100)    def closed(self,spider):        print("spider closed")        self.browser.close()    def start_requests(self):        start_urls = []        with open("target_urls.txt", 'r', encoding='utf-8') as f:            for line in f:                url_id, url = line.split('\t\t')                start_urls.append(url)        for url in start_urls:            yield Request(url=url, callback=self.parse)    def parse(self, response):        yield {            'target_url': response.url,            'comments': response.xpath('//div[@class="comments"]//em//text()').extract()        }以上是我的scrapy代码。我scrapy crawl my_spider -o comments.json用来运行爬虫。您可能会注意到，对于每个 my url，都有一个url_id与之相关的独特之处。如何将每个抓取的结果与url_id. 理想情况下，我想url_id将 .yield 输出结果存储在comments.json.

Answer 1

meta例如，尝试传入参数。我对你的代码做了一些更新：

def start_requests(self):

    with open("target_urls.txt", 'r', encoding='utf-8') as f:

        for line in f:

            url_id, url = line.split('\t\t')

            yield Request(url, self.parse, meta={'url_id': url_id, 'original_url': url})

def parse(self, response):

    yield {

        'target_url': response.meta['original_url'],

        'url_id': response.meta['url_id'],

        'comments': response.xpath('//div[@class="comments"]//em//text()').extract()

    }

Answer 2

回答问题和评论，尝试这样的事情：

from scrapy import Spider, Request

from selenium import webdriver

class MySpider(Spider):

    name = "my_spider"

    def __init__(self):

        self.browser = webdriver.Chrome(executable_path='E:/chromedriver')

        self.browser.set_page_load_timeout(100)

    def closed(self,spider):

        print("spider closed")

        self.browser.close()

    def start_requests(self):

        with open("target_urls.txt", 'r', encoding='utf-8') as f:

            for line in f:

                url_id, url = line.split('\t\t')

                yield Request(url=url, callback=self.parse, meta={'url_id':url_id,'url':url})

    def parse(self, response):

        yield {

        'target_url': response.meta['url'],

        'comments': response.xpath('//div[@class="comments"]//em//text()').extract(),

        'url_id':response.meta['url_id']

    }

正如上一个答案中所说，您可以使用 META ( http://scrapingauthority.com/scrapy-meta )在各种方法之间传递参数。