osmarks/meme-search-engine
1
0
Fork 0
mirror of https://github.com/osmarks/meme-search-engine.git synced 2025-08-04 21:03:47 +00:00
Code Issues Releases Wiki Activity

Fix crawler (rate limit changes)

Browse Source
This commit is contained in:
osmarks 2024-07-15 18:48:55 +01:00
parent 43ff9215fb
commit fc6d0c9409
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
meme-rater/crawler.py
View File

@ -9,6 +9,10 @@ import sys
async def fetch_list_seg(sess, list_url, query): async def fetch_list_seg(sess, list_url, query):
async with sess.get(list_url + ".json", params=query) as res: async with sess.get(list_url + ".json", params=query) as res:
if rate_limit := res.headers.get("x-ratelimit-remaining"):
rl = float(rate_limit)
if rl <= 5.0:
await asyncio.sleep(float(res.headers["x-ratelimit-reset"]))
return await res.json() return await res.json()
async def fetch_past(sess, list_url, n): async def fetch_past(sess, list_url, n):