问刮擦爬行器返回重复值

EN

# -*- coding: utf-8 -*-
import scrapy


class MainSpider(scrapy.Spider):
    name = 'main'
    start_urls = ["https://www.compass.com/agents"]

    def parse(self, response):
        regions = response.xpath('//ul[@class="geoLinks-list textIntent-caption1--strong"]/li')
        for each in regions:
            region_link = each.xpath('.//a/@href').get()
            region_name = each.xpath('.//a/text()').get()

            yield response.follow(url=region_link, callback=self.parse_data, meta={"region_text": region_name})


    def parse_data(self, response):
        region = response.request.meta["region_text"]

        agents = response.xpath('//div[@class="agentCard-contact"]')
        for agent in agents:
            name = agent.xpath('normalize-space(//div[@class="agentCard-contact"]/a/text())').get()
            profile_link = agent.xpath('//div[@class="agentCard-contact"]/a/@href').get()
            email = agent.xpath('//a[@class="textIntent-body agentCard-email"]/@href').get()
            mobile = agent.xpath('//a[@class="textIntent-body agentCard-phone"]/@href').get()

            yield {
                "Name": name,
                "Profile_link": profile_link,
                "Email": email,
                "Mobile": mobile,
                "Region": region,
            }