From 9e0f1d712dcea401923c112a57c1865922b93c75 Mon Sep 17 00:00:00 2001 From: osmarks Date: Sat, 1 Jan 2022 21:28:40 +0000 Subject: [PATCH] 2.0 port, fix horrific reminder problems, drop experimental_qa due to perf --- src/main.py | 4 ++++ src/reminders.py | 13 ++++++++++--- src/search.py | 23 ----------------------- src/telephone.py | 8 ++++---- 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/main.py b/src/main.py index 4a3ec98..6f3d768 100644 --- a/src/main.py +++ b/src/main.py @@ -32,6 +32,10 @@ bot = commands.Bot(command_prefix=commands.when_mentioned_or(config["prefix"]), case_insensitive=True, allowed_mentions=discord.AllowedMentions(everyone=False, users=True, roles=True), intents=intents) bot._skip_check = lambda x, y: False +async def on_guild_available(guild): + logging.info(f"{guild.name}|{guild.id} became available") +bot.on_guild_available = on_guild_available + messages = prometheus_client.Counter("abr_messages", "Messages seen/handled by bot") command_invocations = prometheus_client.Counter("abr_command_invocations", "Total commands invoked (includes failed)") @bot.event diff --git a/src/reminders.py b/src/reminders.py index a1fc387..2db9547 100644 --- a/src/reminders.py +++ b/src/reminders.py @@ -48,7 +48,7 @@ class Reminders(commands.Cog): @commands.command(brief="Set a reminder to be reminded about later.", rest_is_raw=True, help="""Sets a reminder which you will (probably) be reminded about at/after the specified time. All times are UTC unless overridden. - Thanks to new coding and algorithms, reminders are now not done at minute granularity. However, do not expect sub-5s granularity due to miscellaneous latency we have not optimized away. + Thanks to new coding and algorithms, reminders are now not done at minute granularity. However, do not expect sub-5s granularity due to miscellaneous latency which has not been a significant target of optimization. Note that due to technical limitations reminders beyond the year 10000 CE or in the past cannot currently be handled. Note that reminder delivery is not guaranteed, due to possible issues including but not limited to: data loss, me eventually not caring, the failure of Discord (in this case message delivery will still be attempted manually on a case-by-case basis), the collapse of human civilization, or other existential risks.""") async def remind(self, ctx, time, *, reminder): @@ -79,7 +79,7 @@ class Reminders(commands.Cog): def insert_reminder(self, id, time): pos = bisect_left(self.reminder_queue, time, key=lambda x: x[0]) - self.reminder_queue.insert(0, (time, id)) + self.reminder_queue.insert(pos, (time, id)) if pos == 0: self.reminder_event.set() @@ -148,11 +148,18 @@ class Reminders(commands.Cog): await self.bot.database.commit() async def init_reminders(self): - reminders = await self.bot.database.execute_fetchall("SELECT * FROM reminders WHERE expired = 0 AND remind_timestamp < ?", (util.timestamp(),)) + ts = util.timestamp() + # load future reminders + reminders = await self.bot.database.execute_fetchall("SELECT * FROM reminders WHERE expired = 0 AND remind_timestamp > ?", (ts,)) for reminder in reminders: self.insert_reminder(reminder["id"], reminder["remind_timestamp"]) logging.info("Loaded %d reminders", len(reminders)) self.rloop_task = await self.reminder_loop() + # catch reminders which were not fired due to downtime or something + reminders = await self.bot.database.execute_fetchall("SELECT * FROM reminders WHERE expired = 0 AND remind_timestamp <= ?", (ts,)) + logging.info("Firing %d late reminders", len(reminders)) + for reminder in reminders: + await self.fire_reminder(reminder["id"]) async def reminder_loop(self): await self.bot.wait_until_ready() diff --git a/src/search.py b/src/search.py index 5b8b3ec..68f2020 100644 --- a/src/search.py +++ b/src/search.py @@ -9,15 +9,6 @@ import util import io import concurrent.futures -def pool_load_model(model): - from transformers import pipeline - qa_pipeline = pipeline("question-answering", model) - globals()["qa_pipeline"] = qa_pipeline - -def pool_operate(question, context): - qa_pipeline = globals()["qa_pipeline"] - return qa_pipeline(question=question, context=context) - class Parser(html.parser.HTMLParser): def __init__(self): self.links = [] @@ -36,10 +27,6 @@ class Search(commands.Cog): self.wp_search_cache = collections.OrderedDict() self.pool = None - def ensure_pool(self): - if self.pool is not None: return - self.pool = concurrent.futures.ProcessPoolExecutor(max_workers=1, initializer=pool_load_model, initargs=(util.config["ir"]["model"],)) - @commands.command() async def search(self, ctx, *, query): "Search using DuckDuckGo. Returns the first result as a link." @@ -102,16 +89,6 @@ class Search(commands.Cog): file = discord.File(f, "content.txt") await ctx.send(file=file) - @commands.command() - async def experimental_qa(self, ctx, page, *, query): - "Answer questions from the first part of a Wikipedia page, using a finetuned ALBERT model." - self.ensure_pool() - loop = asyncio.get_running_loop() - async with ctx.typing(): - content = await self.wp_fetch(page) - result = await loop.run_in_executor(self.pool, pool_operate, query, content) - await ctx.send("%s (%f)" % (result["answer"].strip(), result["score"])) - def cog_unload(self): asyncio.create_task(self.session.close()) if self.pool is not None: diff --git a/src/telephone.py b/src/telephone.py index ea0972a..7e8852b 100644 --- a/src/telephone.py +++ b/src/telephone.py @@ -75,7 +75,7 @@ class Telephone(commands.Cog): async def send_webhooks(self): while True: webhook, content, username, avatar_url = await self.webhook_queue.get() - wh_obj = discord.Webhook.from_url(webhook, adapter=discord.AsyncWebhookAdapter(self.bot.http._HTTPClient__session)) + wh_obj = discord.Webhook.from_url(webhook, session=self.bot.http._HTTPClient__session) try: await wh_obj.send(content=content, username=username, avatar_url=avatar_url, allowed_mentions=discord.AllowedMentions(everyone=False, roles=False, users=False)) except: @@ -111,7 +111,7 @@ class Telephone(commands.Cog): async def send_to_bridge(self, msg): # discard webhooks and bridge messages (hackily, admittedly, not sure how else to do this) if msg.content == "" and len(msg.attachments) == 0: return - if (msg.author == self.bot.user and msg.content[0] == "<") or msg.author.discriminator == "0000": return + if (msg.author == self.bot.user and (len(msg.content) > 0 and msg.content[0] == "<")) or msg.author.discriminator == "0000": return channel_id = msg.channel.id reply = None if msg.reference: @@ -127,10 +127,10 @@ class Telephone(commands.Cog): except discord.HTTPException: replying_to = None if replying_to: - reply = (eventbus.AuthorInfo(replying_to.author.name, replying_to.author.id, str(replying_to.author.avatar_url), replying_to.author.bot), parse_formatting(self.bot, replying_to.content)) + reply = (eventbus.AuthorInfo(replying_to.author.name, replying_to.author.id, str(replying_to.author.display_avatar.url), replying_to.author.bot), parse_formatting(self.bot, replying_to.content)) else: reply = (None, None) - msg = eventbus.Message(eventbus.AuthorInfo(msg.author.name, msg.author.id, str(msg.author.avatar_url), msg.author.bot), + msg = eventbus.Message(eventbus.AuthorInfo(msg.author.name, msg.author.id, str(msg.author.display_avatar.url), msg.author.bot), parse_formatting(self.bot, msg.content), ("discord", channel_id), msg.id, [ at for at in msg.attachments if not at.is_spoiler() ], reply=reply) await eventbus.push(msg)