如何从一个链接中生成一个已解析的项目，并从同一项目列表中的其他链接生成其他已解析的项目

Question

问题是我一直在从一个地方列表中迭代来刮取纬度经度和海拔高度。问题是当我得到我刮回来的东西时，我无法将它与我当前的 df 链接起来，因为我迭代的名称可能已被修改或跳过。我设法得到了我所看到的名称，但由于它是从其他项目的链接外部解析的，因此无法正常工作。import scrapyimport pandas as pdfrom ..items import latlonglocItemdf = pd.read_csv('wine_df_final.csv')df = df[pd.notnull(df.real_place)]real_place = list(set(df.real_place))class latlonglocSpider(scrapy.Spider):    name = 'latlonglocs'    start_urls = []    for place in real_place:        baseurl =  place.replace(',', '').replace(' ', '+')        cleaned_href = f'http://www.google.com/search?q={baseurl}+coordinates+latitude+longitude+distancesto'        start_urls.append(cleaned_href)    def parse(self, response):        items = latlonglocItem()        items['base_name'] = response.xpath('string(/html/head/title)').get().split(' coordinates')[0]        for href in response.xpath('//*[@id="ires"]/ol/div/h3/a/@href').getall():            if href.startswith('/url?q=https://www.distancesto'):                yield response.follow(href, self.parse_distancesto)            else:                pass        yield items    def parse_distancesto(self, response):        items = latlonglocItem()        try:            items['appellation'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[2]/p/strong)').get()            items['latitude'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[1]/td)').get()            items['longitude'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[2]/td)').get()            items['elevation'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[10]/td)').get()            yield items        except Exception:            pass#output appellation      base_name       elevation    latitude    longitude                  Chalone, USA Santa Cruz, USA                  56.81        35           9.23 发生的事情是我解析了我寻找的内容，然后它进入链接并解析其余信息。但是，显然在我的数据框中，我得到了与其余项目完全无关的名称，即使这样也很难找到匹配项。我希望将信息传递给另一个函数，以便将所有项目一起生成。

Answer 1

这可能有效。我会评论我在做什么和你对我在做什么的理解的一点代码。

import scrapy

import pandas as pd

from ..items import latlonglocItem

df = pd.read_csv('wine_df_final.csv')

df = df[pd.notnull(df.real_place)]

real_place = list(set(df.real_place))

class latlonglocSpider(scrapy.Spider): # latlonglocSpider is a child class of scrapy.Spider

    name = 'latlonglocs'

    start_urls = []

    for place in real_place:

        baseurl =  place.replace(',', '').replace(' ', '+')

        cleaned_href = f'http://www.google.com/search?q={baseurl}+coordinates+latitude+longitude+distancesto'

        start_urls.append(cleaned_href)

    def __init__(self): # Constructor for our class

        # Since we did our own constructor we need to call the parents constructor

        scrapy.Spider.__init__(self)

        self.base_name = None # Here is the base_name we can now use class wide

    def parse(self, response):

        items = latlonglocItem()

        items['base_name'] = response.xpath('string(/html/head/title)').get().split(' coordinates')[0]

        self.base_name = items['base_name'] # Lets store the base_name in the class

        for href in response.xpath('//*[@id="ires"]/ol/div/h3/a/@href').getall():

            if href.startswith('/url?q=https://www.distancesto'):

                yield response.follow(href, self.parse_distancesto)

            else:

                pass

        yield items

    def parse_distancesto(self, response):

        items = latlonglocItem()

        try:

            # If for some reason self.base_name is never assigned in

            # parse() then we want to use an empty string instead of the self.base_name

            # The following syntax means use self.base_name unless it is None or empty

            # in which case just use and empty string.

            base_name = self.base_name or "" # If for some reason

            items['appellation'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[2]/p/strong)').get()

            items['latitude'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[1]/td)').get()

            items['longitude'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[2]/td)').get()

            items['elevation'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[10]/td)').get()

            yield items

        except Exception:

            pass

Answer 2

import scrapy

import pandas as pd

from ..items import latlonglocItem

df = pd.read_csv('wine_df_final.csv')

df = df[pd.notnull(df.real_place)]

real_place = list(set(df.real_place))

class latlonglocSpider(scrapy.Spider): # latlonglocSpider is a child class of scrapy.Spider

    name = 'latlonglocs'

    start_urls = []

    for place in real_place:

        baseurl =  place.replace(',', '').replace(' ', '+')

        cleaned_href = f'http://www.google.com/search?q={baseurl}+coordinates+latitude+longitude+distancesto'

        start_urls.append(cleaned_href)

    def __init__(self): # Constructor for our class

        # Since we did our own constructor we need to call the parents constructor

        scrapy.Spider.__init__(self)

        self.base_name = None # Here is the base_name we can now use class wide

    def parse(self, response):

        for href in response.xpath('//*[@id="ires"]/ol/div/h3/a/@href').getall():

            if href.startswith('/url?q=https://www.distancesto'):

                self.base_name = response.xpath('string(/html/head/title)').get().split(' coordinates')[0]

                yield response.follow(href, self.parse_distancesto)

            else:

                pass

    def parse_distancesto(self, response):

        items = latlonglocItem()

        try:

            # If for some reason self.base_name is never assigned in

            # parse() then we want to use an empty string instead of the self.base_name

            # The following syntax means use self.base_name unless it is None or empty

            # in which case just use and empty string.

            items['base_name'] = self.base_name or "" # If for some reason

            items['appellation'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[2]/p/strong)').get()

            items['latitude'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[1]/td)').get()

            items['longitude'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[2]/td)').get()

            items['elevation'] = response.xpath('string(/html/body/div[3]/div/div[2]/div[3]/div[3]/table/tbody/tr[10]/td)').get()

            yield items

        except Exception:

            pass

并发请求必须设置为 1 才能工作并将 base_name 放置在循环中。