@@ -81,8 +89,10 @@ async def rate(request):
post = await request.post()
meme1 = post["meme1"]
meme2 = post["meme2"]
+ iteration = post["iteration"]
rating = post["rating"]
- await db.execute("INSERT INTO ratings (meme1, meme2, rating) VALUES (?, ?, ?)", (meme1, meme2, rating))
+ await db.execute("INSERT INTO ratings (meme1, meme2, rating, iteration) VALUES (?, ?, ?, ?)", (meme1, meme2, rating, iteration))
+ await db.execute("DELETE FROM queue WHERE meme1 = ? AND meme2 = ?", (meme1, meme2))
await db.commit()
return web.HTTPFound("/")
@@ -93,8 +103,15 @@ CREATE TABLE IF NOT EXISTS ratings (
meme1 TEXT NOT NULL,
meme2 TEXT NOT NULL,
rating TEXT NOT NULL,
+ iteration TEXT,
UNIQUE (meme1, meme2)
);
+CREATE TABLE IF NOT EXISTS queue (
+ meme1 TEXT NOT NULL,
+ meme2 TEXT NOT NULL,
+ iteration TEXT NOT NULL,
+ UNIQUE (meme1, meme2, iteration)
+);
""")
app.router.add_routes(routes)
app.router.add_static("/memes/", "./images")
diff --git a/meme-rater/run_graph.py b/meme-rater/run_graph.py
new file mode 100644
index 0000000..0f152b5
--- /dev/null
+++ b/meme-rater/run_graph.py
@@ -0,0 +1,52 @@
+# claude-3
+
+import json
+import matplotlib.pyplot as plt
+
+# Read data from log.jsonl
+data = []
+with open('log.jsonl', 'r') as file:
+ for line in file:
+ data.append(json.loads(line))
+
+# Extract steps, loss, and val_loss
+steps = [entry['step'] for entry in data if "loss" in entry]
+loss = [entry['loss'] for entry in data if "loss" in entry]
+val_loss_data = [entry['val_loss'] for entry in data if 'val_loss' in entry]
+val_steps = [entry['step'] for entry in data if 'val_loss' in entry]
+
+# Extract individual validation loss series
+val_loss_series = {}
+for val_loss in val_loss_data:
+ for key, value in val_loss.items():
+ if key not in val_loss_series:
+ val_loss_series[key] = []
+ val_loss_series[key].append(value)
+
+# Calculate rolling average for loss
+window_size = 50
+rolling_avg = [sum(loss[i:i+window_size])/window_size for i in range(len(loss)-window_size+1)]
+rolling_steps = steps[window_size-1:]
+
+# Calculate rolling averages for validation loss series
+val_rolling_avgs = {}
+for key, series in val_loss_series.items():
+ val_rolling_avgs[key] = [sum(series[i:i+window_size])/window_size for i in range(len(series)-window_size+1)]
+
+print([(name, min(series)) for name, series in val_loss_series.items()])
+
+# Create the plot
+plt.figure(figsize=(10, 6))
+#plt.plot(steps, loss, label='Loss')
+plt.plot(rolling_steps, rolling_avg, label='Rolling Average (Loss)')
+
+for key, series in val_loss_series.items():
+ #plt.plot(val_steps, series, marker='o', linestyle='', label=f'Validation Loss ({key})')
+ plt.plot(val_steps[window_size-1:], val_rolling_avgs[key], label=f'Rolling Average (Validation Loss {key})')
+
+plt.xlabel('Steps')
+plt.ylabel('Loss')
+plt.title('Loss and Validation Loss vs. Steps')
+plt.legend()
+plt.grid(True)
+plt.show()
diff --git a/meme-rater/shared.py b/meme-rater/shared.py
new file mode 100644
index 0000000..a8398d7
--- /dev/null
+++ b/meme-rater/shared.py
@@ -0,0 +1,53 @@
+import sqlite3
+import hashlib
+from collections import defaultdict
+import numpy
+import random
+
+db = sqlite3.connect("data.sqlite3")
+db.row_factory = sqlite3.Row
+
+val_fraction = 0.2
+def is_val_set(meme1, meme2):
+ def is_one_val(meme):
+ return hashlib.sha256(meme.encode("utf-8")).digest()[0] / 255 < (val_fraction / 2) # not strictly correct but good enough
+ return is_one_val(meme1) or is_one_val(meme2)
+
+def fetch_embedding(filename):
+ csr = db.execute("SELECT embedding_vector FROM files WHERE filename = ?", (filename,))
+ x = numpy.frombuffer(csr.fetchone()[0], dtype="float16")
+ csr.close()
+ return x.copy() # PyTorch complains otherwise due to bad
+
+def map_rating(rating, uncertainty=0.05):
+ match rating:
+ case "1": # meme 1 is better
+ return 1 - uncertainty
+ case "2":
+ return uncertainty
+ case _: raise ValueError("invalid rating, please fix")
+
+def fetch_ratings():
+ trains = defaultdict(list)
+ validations = defaultdict(list)
+ csr = db.execute("SELECT meme1, meme2, rating, iteration FROM ratings")
+ for meme1, meme2, rating, iteration in csr.fetchall():
+ (validations if is_val_set(meme1, meme2) else trains)[int(iteration or "0")].append((fetch_embedding(meme1), fetch_embedding(meme2), map_rating(rating)))
+ csr.close()
+ return list(x[1] for x in sorted(trains.items())), list(x[1] for x in sorted(validations.items()))
+
+def generate_random_permutations(x, n):
+ out = []
+ for _ in range(n):
+ random.shuffle(x)
+ out.append(x.copy())
+ return out
+
+def fetch_all_files():
+ csr = db.execute("SELECT filename, embedding_vector FROM files")
+ x = [ (row[0], numpy.frombuffer(row[1], dtype="float16").copy()) for row in csr.fetchall() ]
+ csr.close()
+ return x
+
+def checkpoint_for(steps):
+ return f"./ckpt/model-{steps}.pt", f"./ckpt/optim-{steps}.pt"
\ No newline at end of file
diff --git a/meme-rater/train.py b/meme-rater/train.py
new file mode 100644
index 0000000..0acde88
--- /dev/null
+++ b/meme-rater/train.py
@@ -0,0 +1,129 @@
+import torch.nn
+import torch.nn.functional as F
+import torch
+import sqlite3
+import random
+import numpy
+import json
+import time
+from tqdm import tqdm
+import math
+from dataclasses import dataclass, asdict
+
+from model import Config as ModelConfig, BradleyTerry
+import shared
+
+trains, validations = shared.fetch_ratings()
+for train, validation in zip(trains, validations):
+ print(len(train), len(validation))
+
+device = "cuda"
+
+@dataclass
+class TrainConfig:
+ model: ModelConfig
+ lr: float
+ weight_decay: float
+ batch_size: int
+ epochs: int
+ compile: bool
+ data_grouped_by_iter: bool
+
+config = TrainConfig(
+ model=ModelConfig(
+ d_emb=1152,
+ n_hidden=1,
+ n_ensemble=16,
+ device=device,
+ dtype=torch.float32,
+ dropout=0.1
+ ),
+ lr=3e-4,
+ weight_decay=0.2,
+ batch_size=1,
+ epochs=5,
+ compile=False,
+ data_grouped_by_iter=False
+)
+
+def exprange(min, max, n):
+ lmin, lmax = math.log(min), math.log(max)
+ step = (lmax - lmin) / (n - 1)
+ return (math.exp(lmin + step * i) for i in range(n))
+
+model = BradleyTerry(config.model)
+params = sum(p.numel() for p in model.parameters())
+print(f"{params/1e6:.1f}M parameters")
+print(model)
+
+optimizer = torch.optim.AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)
+
+def train_step(model, batch, real):
+ optimizer.zero_grad()
+ # model batch
+ win_probabilities = model(batch).float()
+ loss = F.binary_cross_entropy(win_probabilities, real)
+ loss.backward()
+ optimizer.step()
+ loss = loss.detach().cpu().item()
+ return loss
+
+if config.compile:
+ print("compiling...")
+ train_step = torch.compile(train_step)
+
+def batch_from_inputs(inputs: list[tuple[numpy.ndarray, numpy.ndarray, float]]):
+ batch_input = torch.stack([
+ torch.stack([ torch.stack((torch.Tensor(emb1).to(config.model.dtype), torch.Tensor(emb2).to(config.model.dtype))) for emb1, emb2, rating in input ])
+ for input in inputs
+ ]).to(device)
+ target = torch.stack([ torch.Tensor([ rating for emb1, emb2, rating in input ]) for input in inputs ]).to(device)
+ return batch_input, target
+
+def evaluate(steps):
+ print("evaluating...")
+ model.eval()
+ results = {"step": steps, "time": time.time(), "val_loss": {}}
+ for vset, validation in enumerate(validations):
+ with torch.no_grad():
+ batch_input, target = batch_from_inputs([ validation[:128] for _ in range(config.model.n_ensemble) ])
+ result = model(batch_input).float()
+ val_loss = F.binary_cross_entropy(result, target).detach().cpu().item()
+ model.train()
+ results["val_loss"][vset] = val_loss
+ log.write(json.dumps(results) + "\n")
+
+def save_ckpt(log, steps):
+ print("saving...")
+ modelc, optimc = shared.checkpoint_for(steps)
+ torch.save(optimizer.state_dict(), optimc)
+ torch.save(model.state_dict(), modelc)
+
+class JSONEncoder(json.JSONEncoder):
+ def default(self, o):
+ if isinstance(o, torch.dtype):
+ return str(o)
+ else: return super().default(o)
+
+logfile = f"logs/log-{time.time()}.jsonl"
+with open(logfile, "w") as log:
+ steps = 0
+ log.write(JSONEncoder().encode(asdict(config)) + "\n")
+ for epoch in range(config.epochs):
+ for train in (trains if config.data_grouped_by_iter else [[ sample for trainss in trains for sample in trainss ]]):
+ data_orders = shared.generate_random_permutations(train, config.model.n_ensemble)
+ for bstart in range(0, len(train), config.batch_size):
+ batch_input, target = batch_from_inputs([ order[bstart:bstart + config.batch_size] for order in data_orders ])
+ loss = train_step(model, batch_input, target)
+ print(steps, loss)
+ log.write(json.dumps({"loss": loss, "step": steps, "time": time.time()}) + "\n")
+ if steps % 10 == 0:
+ if steps % 250 == 0: save_ckpt(log, steps)
+ loss = evaluate(steps)
+ #print(loss)
+ #best = min(loss, best)
+ steps += 1
+
+ save_ckpt(log, steps)
+
+print(logfile)
\ No newline at end of file