From fc6d0c94091e5c858e424cd284ff697e415099be Mon Sep 17 00:00:00 2001
From: osmarks <osmarks@protonmail.com>
Date: Mon, 15 Jul 2024 18:48:55 +0100
Subject: [PATCH] Fix crawler (rate limit changes)

---
 meme-rater/crawler.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/meme-rater/crawler.py b/meme-rater/crawler.py
index f02bb41..f75d312 100644
--- a/meme-rater/crawler.py
+++ b/meme-rater/crawler.py
@@ -9,6 +9,10 @@ import sys
 
 async def fetch_list_seg(sess, list_url, query):
     async with sess.get(list_url + ".json", params=query) as res:
+        if rate_limit := res.headers.get("x-ratelimit-remaining"):
+            rl = float(rate_limit)
+            if rl <= 5.0:
+                await asyncio.sleep(float(res.headers["x-ratelimit-reset"]))
         return await res.json()
 
 async def fetch_past(sess, list_url, n):