问如果链接保持不变，如何在抓取链接的同时到达下一页？

EN

import scrapy

class EsgKrx1Spider(scrapy.Spider):
    name = 'esg_krx1'
    allowed_domains = ['esg.krx.co.kr']
    start_urls = ['http://esg.krx.co.kr/contents/02/02020000/ESG02020000.jsp']
    custom_settings = {
        "USER_AGENT": 'Mozilla/5.0 (X11; Linux x86_64; rv:97.0) Gecko/20100101 Firefox/97.0'
    }

    def parse(self, response):
        #send a post request to the api
        url = "http://esg.krx.co.kr/contents/99/ESG99000001.jspx"
        
        headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        }

        total_pages = 77
        for page in range(total_pages):
            payload = f"sch_com_nm=&sch_yy=2021&pagePath=%2Fcontents%2F02%2F02020000%2FESG02020000.jsp&code=02%2F02020000%2Fesg02020000&curPage={page+1}"
            yield response.follow(url=url, method='POST', callback=self.parse_result, headers=headers, body=payload)

    def parse_result(self, response):

        # #looping in the result and assigning the company name
        for item in response.json().get('result'):
            yield {
                'company_name': item.get('com_abbrv'),
                'compay_share_id': item.get('isu_cd')
            }

import scrapy
from scrapy_splash import SplashRequest
import json

script = """
function main(splash, args)
  assert(splash:go(args.url))
  assert(splash:wait(7))
  return splash:html()
end
"""
class KorenSiteSpider(scrapy.Spider):
    name = 'k-site'
    start_urls = ['https://esg.krx.co.kr/contents/02/02020000/ESG02020000.jsp']
    custom_settings = {
        'USER_AGENT':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36',
        'DOWNLOAD_DELAY':3
    }

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(
                url = url,
                callback = self.parse, 
                endpoint='execute',
                args = {'lua_source':script}
            )

    def parse(self, response):
        for i in range(1, 78, 1):
            yield scrapy.FormRequest(
                url = 'https://esg.krx.co.kr/contents/99/ESG99000001.jspx',
                method = 'POST',
                formdata = {
                            'sch_com_nm': '',
                            'sch_yy': '2021',
                            'pagePath': '/contents/02/02020000/ESG02020000.jsp',
                            'code': '02/02020000/esg02020000',
                            'curPage': str(i)
                            },
                callback = self.parse_json
            )

    def parse_json(self, response):
        dict_data = json.loads(response.text)

    #looping in the result and assigning the company name
        for i in dict_data['result']:
            company_name = i['com_abbrv']
            company_share_id = i['isu_cd']
            yield {
                'company:name':company_name,
                'company_share_id':company_share_id
            }

2022-02-17 13:55:04 [scrapy.core.scraper] DEBUG: Scraped from <200 https://esg.krx.co.kr/contents/99/ESG99000001.jspx>
{'company:name': '페이퍼코리아', 'company_share_id': '001020'}
2022-02-17 13:55:04 [scrapy.core.scraper] DEBUG: Scraped from <200 https://esg.krx.co.kr/contents/99/ESG99000001.jspx>
{'company:name': '평화산업', 'company_share_id': '090080'}
2022-02-17 13:55:04 [scrapy.core.scraper] DEBUG: Scraped from <200 https://esg.krx.co.kr/contents/99/ESG99000001.jspx>
{'company:name': '평화홀딩스', 'company_share_id': '010770'}
2022-02-17 13:55:04 [scrapy.core.scraper] DEBUG: Scraped from <200 https://esg.krx.co.kr/contents/99/ESG99000001.jspx>
{'company:name': '포스코', 'company_share_id': '005490'}
2022-02-17 13:55:04 [scrapy.core.scraper] DEBUG: Scraped from <200 https://esg.krx.co.kr/contents/99/ESG99000001.jspx>
{'company:name': '포스코강판', 'company_share_id': '058430'}