2.0 port, fix horrific reminder problems, drop experimental_qa due to perf

This commit is contained in:
osmarks 2022-01-01 21:28:40 +00:00
parent ae6830dba5
commit 9e0f1d712d
4 changed files with 18 additions and 30 deletions

View File

@ -32,6 +32,10 @@ bot = commands.Bot(command_prefix=commands.when_mentioned_or(config["prefix"]),
case_insensitive=True, allowed_mentions=discord.AllowedMentions(everyone=False, users=True, roles=True), intents=intents) case_insensitive=True, allowed_mentions=discord.AllowedMentions(everyone=False, users=True, roles=True), intents=intents)
bot._skip_check = lambda x, y: False bot._skip_check = lambda x, y: False
async def on_guild_available(guild):
logging.info(f"{guild.name}|{guild.id} became available")
bot.on_guild_available = on_guild_available
messages = prometheus_client.Counter("abr_messages", "Messages seen/handled by bot") messages = prometheus_client.Counter("abr_messages", "Messages seen/handled by bot")
command_invocations = prometheus_client.Counter("abr_command_invocations", "Total commands invoked (includes failed)") command_invocations = prometheus_client.Counter("abr_command_invocations", "Total commands invoked (includes failed)")
@bot.event @bot.event

View File

@ -48,7 +48,7 @@ class Reminders(commands.Cog):
@commands.command(brief="Set a reminder to be reminded about later.", rest_is_raw=True, help="""Sets a reminder which you will (probably) be reminded about at/after the specified time. @commands.command(brief="Set a reminder to be reminded about later.", rest_is_raw=True, help="""Sets a reminder which you will (probably) be reminded about at/after the specified time.
All times are UTC unless overridden. All times are UTC unless overridden.
Thanks to new coding and algorithms, reminders are now not done at minute granularity. However, do not expect sub-5s granularity due to miscellaneous latency we have not optimized away. Thanks to new coding and algorithms, reminders are now not done at minute granularity. However, do not expect sub-5s granularity due to miscellaneous latency which has not been a significant target of optimization.
Note that due to technical limitations reminders beyond the year 10000 CE or in the past cannot currently be handled. Note that due to technical limitations reminders beyond the year 10000 CE or in the past cannot currently be handled.
Note that reminder delivery is not guaranteed, due to possible issues including but not limited to: data loss, me eventually not caring, the failure of Discord (in this case message delivery will still be attempted manually on a case-by-case basis), the collapse of human civilization, or other existential risks.""") Note that reminder delivery is not guaranteed, due to possible issues including but not limited to: data loss, me eventually not caring, the failure of Discord (in this case message delivery will still be attempted manually on a case-by-case basis), the collapse of human civilization, or other existential risks.""")
async def remind(self, ctx, time, *, reminder): async def remind(self, ctx, time, *, reminder):
@ -79,7 +79,7 @@ class Reminders(commands.Cog):
def insert_reminder(self, id, time): def insert_reminder(self, id, time):
pos = bisect_left(self.reminder_queue, time, key=lambda x: x[0]) pos = bisect_left(self.reminder_queue, time, key=lambda x: x[0])
self.reminder_queue.insert(0, (time, id)) self.reminder_queue.insert(pos, (time, id))
if pos == 0: if pos == 0:
self.reminder_event.set() self.reminder_event.set()
@ -148,11 +148,18 @@ class Reminders(commands.Cog):
await self.bot.database.commit() await self.bot.database.commit()
async def init_reminders(self): async def init_reminders(self):
reminders = await self.bot.database.execute_fetchall("SELECT * FROM reminders WHERE expired = 0 AND remind_timestamp < ?", (util.timestamp(),)) ts = util.timestamp()
# load future reminders
reminders = await self.bot.database.execute_fetchall("SELECT * FROM reminders WHERE expired = 0 AND remind_timestamp > ?", (ts,))
for reminder in reminders: for reminder in reminders:
self.insert_reminder(reminder["id"], reminder["remind_timestamp"]) self.insert_reminder(reminder["id"], reminder["remind_timestamp"])
logging.info("Loaded %d reminders", len(reminders)) logging.info("Loaded %d reminders", len(reminders))
self.rloop_task = await self.reminder_loop() self.rloop_task = await self.reminder_loop()
# catch reminders which were not fired due to downtime or something
reminders = await self.bot.database.execute_fetchall("SELECT * FROM reminders WHERE expired = 0 AND remind_timestamp <= ?", (ts,))
logging.info("Firing %d late reminders", len(reminders))
for reminder in reminders:
await self.fire_reminder(reminder["id"])
async def reminder_loop(self): async def reminder_loop(self):
await self.bot.wait_until_ready() await self.bot.wait_until_ready()

View File

@ -9,15 +9,6 @@ import util
import io import io
import concurrent.futures import concurrent.futures
def pool_load_model(model):
from transformers import pipeline
qa_pipeline = pipeline("question-answering", model)
globals()["qa_pipeline"] = qa_pipeline
def pool_operate(question, context):
qa_pipeline = globals()["qa_pipeline"]
return qa_pipeline(question=question, context=context)
class Parser(html.parser.HTMLParser): class Parser(html.parser.HTMLParser):
def __init__(self): def __init__(self):
self.links = [] self.links = []
@ -36,10 +27,6 @@ class Search(commands.Cog):
self.wp_search_cache = collections.OrderedDict() self.wp_search_cache = collections.OrderedDict()
self.pool = None self.pool = None
def ensure_pool(self):
if self.pool is not None: return
self.pool = concurrent.futures.ProcessPoolExecutor(max_workers=1, initializer=pool_load_model, initargs=(util.config["ir"]["model"],))
@commands.command() @commands.command()
async def search(self, ctx, *, query): async def search(self, ctx, *, query):
"Search using DuckDuckGo. Returns the first result as a link." "Search using DuckDuckGo. Returns the first result as a link."
@ -102,16 +89,6 @@ class Search(commands.Cog):
file = discord.File(f, "content.txt") file = discord.File(f, "content.txt")
await ctx.send(file=file) await ctx.send(file=file)
@commands.command()
async def experimental_qa(self, ctx, page, *, query):
"Answer questions from the first part of a Wikipedia page, using a finetuned ALBERT model."
self.ensure_pool()
loop = asyncio.get_running_loop()
async with ctx.typing():
content = await self.wp_fetch(page)
result = await loop.run_in_executor(self.pool, pool_operate, query, content)
await ctx.send("%s (%f)" % (result["answer"].strip(), result["score"]))
def cog_unload(self): def cog_unload(self):
asyncio.create_task(self.session.close()) asyncio.create_task(self.session.close())
if self.pool is not None: if self.pool is not None:

View File

@ -75,7 +75,7 @@ class Telephone(commands.Cog):
async def send_webhooks(self): async def send_webhooks(self):
while True: while True:
webhook, content, username, avatar_url = await self.webhook_queue.get() webhook, content, username, avatar_url = await self.webhook_queue.get()
wh_obj = discord.Webhook.from_url(webhook, adapter=discord.AsyncWebhookAdapter(self.bot.http._HTTPClient__session)) wh_obj = discord.Webhook.from_url(webhook, session=self.bot.http._HTTPClient__session)
try: try:
await wh_obj.send(content=content, username=username, avatar_url=avatar_url, allowed_mentions=discord.AllowedMentions(everyone=False, roles=False, users=False)) await wh_obj.send(content=content, username=username, avatar_url=avatar_url, allowed_mentions=discord.AllowedMentions(everyone=False, roles=False, users=False))
except: except:
@ -111,7 +111,7 @@ class Telephone(commands.Cog):
async def send_to_bridge(self, msg): async def send_to_bridge(self, msg):
# discard webhooks and bridge messages (hackily, admittedly, not sure how else to do this) # discard webhooks and bridge messages (hackily, admittedly, not sure how else to do this)
if msg.content == "" and len(msg.attachments) == 0: return if msg.content == "" and len(msg.attachments) == 0: return
if (msg.author == self.bot.user and msg.content[0] == "<") or msg.author.discriminator == "0000": return if (msg.author == self.bot.user and (len(msg.content) > 0 and msg.content[0] == "<")) or msg.author.discriminator == "0000": return
channel_id = msg.channel.id channel_id = msg.channel.id
reply = None reply = None
if msg.reference: if msg.reference:
@ -127,10 +127,10 @@ class Telephone(commands.Cog):
except discord.HTTPException: except discord.HTTPException:
replying_to = None replying_to = None
if replying_to: if replying_to:
reply = (eventbus.AuthorInfo(replying_to.author.name, replying_to.author.id, str(replying_to.author.avatar_url), replying_to.author.bot), parse_formatting(self.bot, replying_to.content)) reply = (eventbus.AuthorInfo(replying_to.author.name, replying_to.author.id, str(replying_to.author.display_avatar.url), replying_to.author.bot), parse_formatting(self.bot, replying_to.content))
else: else:
reply = (None, None) reply = (None, None)
msg = eventbus.Message(eventbus.AuthorInfo(msg.author.name, msg.author.id, str(msg.author.avatar_url), msg.author.bot), msg = eventbus.Message(eventbus.AuthorInfo(msg.author.name, msg.author.id, str(msg.author.display_avatar.url), msg.author.bot),
parse_formatting(self.bot, msg.content), ("discord", channel_id), msg.id, [ at for at in msg.attachments if not at.is_spoiler() ], reply=reply) parse_formatting(self.bot, msg.content), ("discord", channel_id), msg.id, [ at for at in msg.attachments if not at.is_spoiler() ], reply=reply)
await eventbus.push(msg) await eventbus.push(msg)