Ограничение парсинга телеграмма
Есть задача - спарсить n-ое количество сообщений из телеграмм каналов. Использовал такой скрипт:
import random
import requests
import xlrd
import psycopg2
connection = psycopg2.connect(user="postgres",
password="root",
host="127.0.0.1",
port="5432",
database="botgethist")
cursor = connection.cursor()
data = xlrd.open_workbook("D:/RED.xlsx")
data = data.sheet_by_index(0)
lkdt = []
lkcol = [0, 1, 3]
for v in range(0, data.nrows):
lkp = []
for pks in lkcol:
lkp.append(str(data.cell_value(v, pks)).strip())
lkp.insert(2, "ru")
lkdt.append(lkp)
request_delay = 2
requests_per_minute = 30
seconds_per_minute = 60
last_request_time = time.time()
proxies_list = [
{"http": "http://Qvnb7L:[email protected]:10061"},
{"http": "http://Qvnb7L:[email protected]:10060"},
{"http": "http://Qvnb7L:[email protected]:10059"},
{"http": "http://Qvnb7L:[email protected]:10058"},
{"http": "http://Qvnb7L:[email protected]:10057"}
]
token = "1779257401:AAHNOrRslVlNtTHxkb45uJgxWro-dvaLBrs"
for k in range(len(lkdt)):
try:
current_time = time.time()
time_since_last_request = current_time - last_request_time
if time_since_last_request < seconds_per_minute / requests_per_minute:
time.sleep(seconds_per_minute / requests_per_minute - time_since_last_request)
sk = lkdt[k]
ids = "@" + str(sk[1]).split("/")[-1]
url = "https://api.telegram.org/bot{0}/getChat?chat_id={1}".format(token, ids)
s = requests.post(url, proxies=random.choice(proxies_list)).json()
if str(s["ok"]) == "True":
sk.append(int(s["result"]["id"]))
lkdt[k] = tuple(sk)
cursor.execute("INSERT INTO tgchannels (tgchannels_name, tgchannels_url, tgchannels_lang, tgchannels_country, tgchannels_idchan) VALUES (%s, %s, %s, %s, %s)", lkdt[k])
print(str(k + 1) + " " + str(len(lkdt))
else:
print(s)
if str(s['description']).startswith("Too Many Requests:") == True:
print(str(k + 1) + " " + str(len(lkdt)) + " Error : many requests")
time.sleep(10)
if str(s['description']) == "Bad Request: chat not found":
print(str(k + 1) + " " + str(len(lkdt)) + " Error : chat not found")
time.sleep(10)
except Exception as e:
print(str(e))
print(sk)
time.sleep(30)
connection.commit()
cursor.close()
connection.close()
Но постоянно утыкаюсь в ограничение после 206 канала. Что делать? Какие варианты? Может через телетон переписать?