diff --git a/.gitignore b/.gitignore index 1440ad9..7517520 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,13 @@ code-guessing/analytics/people */counts-old.csv */*.png *.zst +tacnet.sqlite3 +basic +basic.png +complex +complex.png +minecraft_images +comparisons.jsonl +nouns_cache.db +nouns.json +*.scad diff --git a/README.md b/README.md index 495c979..c35853a 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,6 @@ This comes with absolutely no guarantee of support or correct function, although * `bmp280_prometheus.py` - read BMP280 temperature/pressure sensors and export as Prometheus metrics. * `captive_portal.py` - automatically fill in captive portal forms (WIP). * `scd4x_prometheus.py` - read SCD4x temperature/humidity/CO2 sensors and export as Prometheus metrics. -* `weight_painter.py` - paint arbitrary images into neural network weight matrices. Uses a permutation, so the distribution is preserved so training dynamics remain unaffected (so long as the network doesn't care about organization below the weight matrix level - this is not safe for attention heads etc). +* `weight_painter.py` - paint arbitrary images into neural network weight matrices. Uses a permutation, so the distribution is preserved so training dynamics should not be affected much. * `cool-effect.glsl` - a nice effect I made by accident whilst trying to make a cooler one. * `portable_monitor_wallmount.py` - very simple CADQuery script which generates a frame for my portable monitor's top/bottom bezels so it can be stuck to the wall and slid out easily. diff --git a/aidans_bad_code.py b/aidans_bad_code.py index bd84774..80b1619 100644 --- a/aidans_bad_code.py +++ b/aidans_bad_code.py @@ -1,7 +1,7 @@ class Primes: def __init__(self, max): self.internal = range(2,max+1) - + def __next__(self): i = self.internal.__iter__().__next__() self.internal = filter(lambda n : n % i != 0, self.internal) @@ -10,4 +10,4 @@ class Primes: def __iter__(self): return self for i in Primes(100): - print(i) \ No newline at end of file + print(i) diff --git a/autocrafter.py b/autocrafter.py new file mode 100644 index 0000000..6171ed6 --- /dev/null +++ b/autocrafter.py @@ -0,0 +1,260 @@ +from dataclasses import dataclass +from collections import Counter, defaultdict, deque +import math +from typing import Generator +import scipy.optimize as opt +import numpy as np +import graphviz + +@dataclass +class Recipe: + slots: list[str] + quantity: int + processing: bool + +short_names = { + "minecraft:stone": "st", + "minecraft:redstone": "re", + "minecraft:glass": "gl", + "minecraft:glass_pane": "gp", + "minecraft:gold_ingot": "gi", + "minecraft:iron_ingot": "ii", + "minecraft:oak_wood": "ow", + "minecraft:oak_planks": "op", + "minecraft:oak_chest": "oc", + "computercraft:computer": "cc", + "computercraft:computer_advanced": "ca", + "computercraft:turtle": "tu", + "minecraft:birch_wood": "bw", + "minecraft:birch_planks": "bp", + "quark:birch_chest": "bc", + "computercraft:turtle_advanced": "ta", + "minecraft:gold_block": "gb", + "minecraft:coal": "co", + "minecraft:charcoal": "ch", + "minecraft:gold_ore": "go", + "minecraft:iron_ore": "io", + "minecraft:sand": "sa", + None: "-" +} + +short_names_inv = {v: k for k, v in short_names.items()} + +recipes = {} + +def recipe_short(output, qty, inputs, processing=False): + recipes[short_names_inv[output]] = Recipe([short_names_inv.get(slot, slot) for slot in inputs.split()], qty, processing) + +recipe_short("gp", 16, "gl gl gl gl gl gl") +recipe_short("op", 4, "ow") +recipe_short("oc", 1, "op op op op - op op op op") +recipe_short("cc", 1, "st st st st re st st gp st") +recipe_short("tu", 1, "ii ii ii ii cc ii ii oc ii") +recipe_short("ca", 1, "gi gi gi gi re gi gi gp gi") +recipe_short("ta", 1, "gi gi gi gi ca gi gi oc gi") + +@dataclass +class Inventory: + contents: dict[str, int] + + def __getitem__(self, item): + return self.contents.get(item, 0) + + def add(self, item, quantity): + new_inventory = self.contents.copy() + new_inventory[item] = self.contents.get(item, 0) + quantity + return Inventory(new_inventory) + + def take(self, item, quantity): + return self.add(item, -quantity) + +class NoRecipe(BaseException): + pass + +def solve(item: str, quantity: int, inventory: Inventory, use_callback, craft_callback) -> Inventory: + directly_available = min(inventory[item], quantity) # Consume items from storage if available + if directly_available > 0: + use_callback(item, directly_available) + inventory = inventory.take(item, directly_available) + quantity -= directly_available + + if quantity > 0: + if recipe := recipes.get(item): + recipe_runs = math.ceil(quantity / recipe.quantity) + for citem, cquantity in Counter(recipe.slots).items(): + if citem is not None: + inventory = solve(citem, recipe_runs * cquantity, inventory, use_callback, craft_callback) # Recurse into subrecipe + craft_callback(recipe, item, recipe_runs) + inventory = inventory.add(item, recipe_runs * recipe.quantity - quantity) # Add spare items to tracked inventory + else: + raise NoRecipe(item, quantity) # We need to make this and can't + + return inventory + +final = solve("computercraft:turtle", 1, Inventory({ + "minecraft:stone": 100, + "minecraft:redstone": 1, + "minecraft:iron_ingot": 10, + "minecraft:oak_wood": 2, + "minecraft:glass": 7 +}), lambda item, quantity: print(f"Using {quantity} {item}"), lambda recipe, item, runs: print(f"Crafting {runs}x{recipe.quantity} {item}")) + +print(final) + +def compute_item_graph(recipes_general): + recipes_forward_graph = defaultdict(set) # edge u→v exists where u is used to craft v + recipes_backward_graph = defaultdict(set) # edge u→v exists where v's recipe contains u + for src, recipes in recipes_general.items(): + for recipe in recipes: + for input in Counter(recipe.slots).keys(): + if input is not None: + recipes_forward_graph[input].add(src) + recipes_backward_graph[src].add(input) + + return recipes_forward_graph, recipes_backward_graph + +def item_graph(recipe_forward_graph, name): + dot = graphviz.Digraph("items", format="png", body=["\tpad=0.5\n"]) + dot.attr("graph", nodesep="1", ranksep="1") + nodes = set() + + def mk_node(item): + if not item in nodes: + dot.node(item.replace(":", "_"), "", image=f"minecraft_images/{item}.png", imagescale="true", shape="plaintext") + nodes.add(item) + + for input, outputs in recipe_forward_graph.items(): + mk_node(input) + for output in outputs: + dot.edge(input.replace(":", "_"), output.replace(":", "_")) + mk_node(output) + + dot.render(filename=name) + +recipes_general = defaultdict(list) + +for src, recipe in recipes.items(): + recipes_general[src].append(recipe) + +item_graph(compute_item_graph(recipes_general)[0], "basic") + +# Add multiple recipes for things to make the ILP solver work harder. + +def recipe_short_extra(output, qty, inputs, processing=False): + recipes_general[short_names_inv[output]].append(Recipe([short_names_inv.get(slot, slot) for slot in inputs.split()], qty, processing)) + +recipe_short_extra("bp", 4, "bw") +recipe_short_extra("bc", 1, "bp bp bp bp - bp bp bp bp") +recipe_short_extra("tu", 1, "ii ii ii ii cc ii ii bc ii") +recipe_short_extra("gb", 1, "gi gi gi gi gi gi gi gi gi") +recipe_short("ta", 1, "gi gi gi gi ca gi gi bc gi") +recipe_short_extra("ta", 1, "gi gb gi gi tu gi - gi -") +recipe_short_extra("ta", 1, "gi gb gi gi tu gi - gi -") +for count in [1, 8]: + recipe_short_extra("gl", count, "ch" + " sa" * count, processing=True) + recipe_short_extra("gl", count, "co" + " sa" * count, processing=True) + recipe_short_extra("ii", count, "ch" + " io" * count, processing=True) + recipe_short_extra("ii", count, "co" + " io" * count, processing=True) + recipe_short_extra("gi", count, "ch" + " go" * count, processing=True) + recipe_short_extra("gi", count, "co" + " go" * count, processing=True) + recipe_short_extra("ch", count, "co" + " ow" * count, processing=True) + recipe_short_extra("ch", count, "co" + " bw" * count, processing=True) + +recipes_forward_graph, recipes_backward_graph = compute_item_graph(recipes_general) + +item_graph(recipes_forward_graph, "complex") + +def topo_sort_inputs(item: str) -> Generator[str]: + seen = set() + + # DFS to find root nodes in relevant segment (no incoming edges → no recipes) + def dfs(item): + if item in seen or not item: return + seen.add(item) + + for input in recipes_backward_graph[item]: + dfs(input) + + dfs(item) + roots = deque(item for item in seen if len(recipes_general[item]) == 0) + + # Kahn's algorithm (approximately) + # Count incoming edges + counts = { item: len(inputs) for item, inputs in recipes_backward_graph.items() if item in seen } + + while roots: + item = roots.popleft() + yield item + for out_edge in recipes_forward_graph[item]: + # Filter out items not in current operation's subtree + if (count := counts.get(out_edge)) != None: + counts[out_edge] -= 1 + # It's now safe to use this item since its dependencies are all in output + if counts[out_edge] == 0: + roots.append(out_edge) + +def solve_ilp(item: str, quantity: int, inventory: Inventory, use_callback, craft_callback): + sequence = list(topo_sort_inputs(item)) + + recipe_steps = [] + + # Rewrite (involved) recipes as production/consumption numbers. + items = { sitem: [] for sitem in sequence } + + for item in sequence: + for recipe in recipes_general[item]: + step_changes = { sitem: 0 for sitem in sequence } + for citem, cquantity in Counter(recipe.slots).items(): + if citem is not None: + step_changes[citem] = -cquantity + step_changes[item] = recipe.quantity + + recipe_steps.append((item, recipe)) + + for sitem, coef in step_changes.items(): + items[sitem].append(coef) + + objective = np.ones(len(recipe_steps)) + # The amount of each item we produce/consume is linearly dependent on how many times each recipe is executed. + # This matrix is that linear transform. + # Solver wants upper bounds so flip signs. + production_matrix = -np.stack([np.array(coefs) for item, coefs in items.items()]) + # production_matrix @ x is the vector of item consumption, so we upper-bound that with inventory item counts + # and require that we produce the required output (negative net consumption) + item_constraint_vector = np.array([ -quantity + inventory[i] if i == item else inventory[i] for i in sequence ]) + + soln = opt.linprog(objective, integrality=np.ones_like(objective), A_ub=production_matrix, b_ub=item_constraint_vector) + + match soln.status: + case 0: + print("OK") + # soln.x is now the number of times to execute each recipe_step + item_consumption = production_matrix @ soln.x + for item_name, consumption in zip(sequence, item_consumption): + consumption = int(consumption) + if consumption > 0: + use_callback(item_name, consumption) + inventory = inventory.take(item_name, consumption) + for (recipe_output, recipe_spec), execution_count in zip(recipe_steps, soln.x): + execution_count = int(execution_count) + if execution_count > 0: + craft_callback(recipe_spec, recipe_output, execution_count) + return inventory + case 1: + print("iteration limit reached") + raise NoRecipe + case 2: + print("infeasible") + raise NoRecipe + +print(solve_ilp("computercraft:turtle_advanced", 1, Inventory({ + "minecraft:stone": 100, + "minecraft:redstone": 1, + "minecraft:iron_ingot": 10, + "minecraft:gold_ore": 16, + "minecraft:oak_wood": 3, + "minecraft:birch_wood": 8, + "minecraft:glass": 7, + "minecraft:coal": 1, + "minecraft:sand": 16, +}), lambda item, quantity: print(f"Using {quantity} {item}"), lambda recipe, item, runs: print(f"Crafting {runs}x{recipe.quantity} {item}"))) diff --git a/bigram.py b/bigram.py new file mode 100644 index 0000000..baf79e7 --- /dev/null +++ b/bigram.py @@ -0,0 +1,25 @@ +from PIL import Image +import sys +from collections import defaultdict +import math + +out = Image.new("RGB", (256, 256)) + +ctr = defaultdict(lambda: 0) + +BS = 2<<18 + +with open(sys.argv[2], "rb") as f: + last = b"" + while xs := f.read(BS): + for a, b in zip(last + xs, last + xs[1:]): + ctr[a, b] += 1 + last = bytes([xs[-1]]) + +ctrl = { k: math.log(v) for k, v in ctr.items() } +maxv = max(ctrl.values()) +for x, y in ctrl.items(): + s = int(y / maxv * 255) + out.putpixel((x[0], x[1]), (0, s, 0)) + +out.save(sys.argv[1]) diff --git a/captive_portal.py b/captive_portal.py index c72747e..0985e1f 100644 --- a/captive_portal.py +++ b/captive_portal.py @@ -42,7 +42,7 @@ session = requests.Session() DETECTPORTAL_URL = "http://detectportal.firefox.com/canonical.html" DETECTPORTAL_CONTENT = '' -PRIORITY_KEYWORDS = {"registr", "login", "signup", "signin"} +PRIORITY_KEYWORDS = {"regist", "login", "signup", "signin"} CONFIRM_SUFFIXES = {"2", "repeat", "confirm", "_repeat", "_confirm"} EMAIL_BASE = "0t.lt" @@ -85,11 +85,20 @@ def handle_response(response): queue_ext = [] for link in soup.find_all("a"): if href := link.get("href"): - href = urllib.parse.urljoin(response.url, href) if is_priority(href): queue_ext.insert(0, href) else: queue_ext.append(href) + """ + for script in soup.find_all("script"): + if src := script.get("src"): + queue_ext.append(src) + """ + for meta in soup.find_all("meta"): + if meta.get("http-equiv", "").lower() == "refresh": + if content := meta.get("content"): + if mat := re.match(r"\d+;URL='(.*)'", content): + queue_ext.append(mat.group(1)) for form in soup.find_all("form"): fields = {} @@ -152,6 +161,7 @@ def handle_response(response): response = session.post(action, data=fields) handle_response(response) + queue_ext = [ urllib.parse.urljoin(response.url, q) for q in queue_ext ] queue.extend(x for x in queue_ext if x not in tried) while True: diff --git a/compare_things.py b/compare_things.py new file mode 100644 index 0000000..2e5942a --- /dev/null +++ b/compare_things.py @@ -0,0 +1,23 @@ +import json +import random + +with open("nouns.json", "r") as f: + nouns = set(json.load(f)) + +with open("comparisons.jsonl", "a") as f: + def writeline(obj): + f.write(json.dumps(obj, separators=(",", ":")) + "\n") + + for noun in nouns: + other_noun = random.choice(list(nouns - {noun})) + print(noun, "/",other_noun) + pref = input("a/b/e/x/y: ") + writeline({"a": noun, "b": other_noun, "pref": pref}) + if pref == "x": + writeline({"a": noun, "b": other_noun, "pref": "x"}) + nouns.remove(noun) + elif pref == "y": + writeline({"a": other_noun, "b": noun, "pref": "y"}) + nouns.remove(other_noun) + + f.flush() diff --git a/extract_nouns.py b/extract_nouns.py new file mode 100644 index 0000000..5d4734f --- /dev/null +++ b/extract_nouns.py @@ -0,0 +1,53 @@ +import openai +import json +import os +import shelve +import re +import random + +openai.api_key = os.environ["OPENAI_API_KEY"] + +client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"]) + +def chunks(text, size): + out = [""] + for line in text.split("\n"): + out[-1] += line + "\n" + if len(out[-1]) > size: + out.append("") + return [ x.removesuffix("\n") for x in out if x ] + +def extract_nouns(text): + completion = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": f"""Extract all unique simple noun phrases from this document and put them in a JSON array in the singular: +``` +{text} +```"""}], + response_format={"type": "json_object"}, + max_tokens=16384, + temperature=0.2 # should be 0 but repetition issues at 0 + ) + result = json.loads(completion.choices[0].message.content) + return result[next(iter(result.keys()))] + +with open("../website/strings.json", "r") as f: + strings = json.load(f) + +nouns = set() +with shelve.open("nouns_cache.db") as db: + for bigstring in strings: + for string in chunks(bigstring, 8192): + if nouns: print(random.choices(list(nouns), k=10)) + if string in db: + nouns.update(db[string]) + else: + print("reading:", string[:100]) + s_nouns = extract_nouns(string) + nouns.update(s_nouns) + print(len(s_nouns), "/", len(nouns)) + db[string] = s_nouns + +with open("nouns.json", "w") as f: + json.dump(list(nouns), f) diff --git a/memeticize.py b/memeticize.py index ab82f5d..0c43af1 100644 --- a/memeticize.py +++ b/memeticize.py @@ -1,9 +1,9 @@ import os, sys, subprocess, datetime -dt_threshold = datetime.datetime(2023, 6, 16).timestamp() +dt_threshold = datetime.datetime(2024, 10, 22).timestamp() _, indir, outdir = sys.argv -for x in os.listdir(indir): +for x in sorted(os.listdir(indir)): inpath = os.path.join(indir, x) if os.stat(inpath).st_mtime > dt_threshold: if subprocess.run(("feh", inpath)).returncode == 0: diff --git a/parametric_mesh.py b/parametric_mesh.py new file mode 100644 index 0000000..8c2b721 --- /dev/null +++ b/parametric_mesh.py @@ -0,0 +1,18 @@ +from solid2 import * +import numpy as np + +THICKNESS = 3 +XYSIZE = 58 +HOLE_SIZE = 4 +LINE_SIZE = 0.5 + +LINE_SPACING = HOLE_SIZE + LINE_SIZE +XYSIZE = (XYSIZE // LINE_SPACING) * LINE_SPACING + LINE_SIZE +print(XYSIZE) + +xlines = [ cube(LINE_SIZE, XYSIZE, THICKNESS).translate(x, 0, 0) for x in np.arange(0, XYSIZE, LINE_SPACING) ] +ylines = [ cube(XYSIZE, LINE_SIZE, THICKNESS).translate(0, y, 0) for y in np.arange(0, XYSIZE, LINE_SPACING) ] + +model = union()(*xlines).union()(*ylines) + +model.save_as_scad() diff --git a/smtp2rss.py b/smtp2rss.py index f77c40b..40473cb 100644 --- a/smtp2rss.py +++ b/smtp2rss.py @@ -99,7 +99,10 @@ def clean_html(html): remove_unknown_tags=True, safe_attrs_only=True ) - return cleaner.clean_html(feedparser.sanitizer._sanitize_html(html.replace("", ""), "utf-8", "text/html")) + try: + return cleaner.clean_html(feedparser.sanitizer._sanitize_html(html.replace("", ""), "utf-8", "text/html")) + except: + return "HTML parse error" def email_to_html(emsg, debug_info=False): if isinstance(emsg, Message): diff --git a/wfc.html b/wfc.html index d61a3d4..71e19b5 100644 --- a/wfc.html +++ b/wfc.html @@ -1,40 +1,140 @@ - - - + + + +