Как сделать ответ в виде text при запросе с помощью grequests
Код:
import grequests
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from openpyxl import Workbook
import time
start_time = time.time()
ua = UserAgent()
cookies = {
'ASP.NET_SessionId': 'i1gip0fre5uzl4iqlkubv1cp',
'SLG_G_WPT_TO': 'ru',
'SLG_GWPT_Show_Hide_tmp': '1',
'SLG_wptGlobTipTmp': '1',
'ICusrcartgd': 'be6d8ad2-c52e-49b8-83b2-f384a9feaa60',
'IWusrsesckgd': 'jojhbQMjYWEdV9ohRKijJKalgxKEvPEPzVqoH/F2376n50ziaNRcMA==',
}
headers = {
'authority': 'catalog.aquamarine.kz',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Opera GX";v="85"',
'accept': 'application/json, text/javascript, */*; q=0.01',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'x-requested-with': 'XMLHttpRequest',
'sec-ch-ua-mobile': '?0',
'user-agent': ua.random,
'sec-ch-ua-platform': '"Windows"',
'origin': 'https://catalog.aquamarine.kz',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://catalog.aquamarine.kz/catalog/index.aspx',
'accept-language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
# Requests sorts cookies= alphabetically
# 'cookie': 'ASP.NET_SessionId=i1gip0fre5uzl4iqlkubv1cp; SLG_G_WPT_TO=ru; SLG_GWPT_Show_Hide_tmp=1; SLG_wptGlobTipTmp=1; ICusrcartgd=be6d8ad2-c52e-49b8-83b2-f384a9feaa60; IWusrsesckgd=jojhbQMjYWEdV9ohRKijJKalgxKEvPEPzVqoH/F2376n50ziaNRcMA==',
}
data = {
'msearch': '',
}
def get_free_proxies():
url = "https://free-proxy-list.net/"
# получаем ответ HTTP и создаем объект soup
soup = BeautifulSoup(requests.get(url).content, "lxml")
proxies = []
for row in soup.find("table", class_='table').find_all("tr"):
tds = row.find_all("td")
try:
ip = tds[0].text.strip()
port = tds[1].text.strip()
host = f"{ip}:{port}"
proxies.append(host)
except IndexError:
continue
return proxies
free_proxies = get_free_proxies()
sites = []
for page in range(1,287):
sites.append(f'https://catalog.aquamarine.kz/catalog/products.ashx?rnd=536811965&q=&spec=&mip=317&map=7777%20777&mippg=161&mappg=5466%20222&miw=0.14&maw=137.74&miq=1&maq=241&miprcs=999999.999&maprcs=0&page={page}&sort=art-down&view=2&spc=1,&brid=7')
response = (grequests.post(url, cookies=cookies, data=data, headers=headers, proxies = free_proxies) for url in sites)
a = grequests.map(response)
for i in a:
print(i.text)
print(f'Время выполнения: {time.time() - start_time} секунд')
Мне нужно получить ответ в виде text. И да я понимаю что grequests.map вернёт просто статус, но я не знаю как тогда делать.