From 7424beaa2dfb24886ae8ddf0c8abdce331ab8fe4 Mon Sep 17 00:00:00 2001 From: osmarks Date: Fri, 29 Jan 2021 23:25:09 +0000 Subject: [PATCH] slightly faster invalidation via size tracking --- generate_manifest.py | 14 +++++++++++--- manifest | 4 ++-- src/main.lua | 5 +++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/generate_manifest.py b/generate_manifest.py index 1eabe1c..78859e5 100755 --- a/generate_manifest.py +++ b/generate_manifest.py @@ -22,8 +22,11 @@ if os.path.exists("./manifest"): def hash_file(path): file = open(path, "rb") h = hashlib.sha256() - while data := file.read(65536): h.update(data) - return h.hexdigest() + count = 0 + while data := file.read(65536): + h.update(data) + count += len(data) + return h.hexdigest(), count if args.sign: print("Signing update") @@ -33,16 +36,21 @@ if args.sign: open("dist/update-key.hex", "w").write(pubkey) files = dict() +sizes = dict() code = Path("./dist/") for path in code.glob("**/*"): if not path.is_dir(): - files["/".join(path.parts[1:])] = hash_file(path) + hexhash, count = hash_file(path) + mpath = "/".join(path.parts[1:]) + files[mpath] = hexhash + sizes[mpath] = count def deterministic_json_serialize(x): return json.dumps(x, sort_keys=True, separators=(",", ":")) manifest_data = deterministic_json_serialize({ "files": files, + "sizes": sizes, "timestamp": int(datetime.datetime.now().timestamp()), "build": counter + 1, "description": args.description diff --git a/manifest b/manifest index c955e34..f916375 100644 --- a/manifest +++ b/manifest @@ -1,2 +1,2 @@ -{"build":180,"description":"fix info","files":{"LICENSES":"f3549d84d66eb53dd4a421a4341d77d3d217c1b117d67e3be8f5211adcda0952","autorun.lua":"9392777418bbd861836a95c0df6b254a914d861c4d7d73baa48867e4f7f2a8e9","bin/5rot26.lua":"91b66cd6d4b33081b25c456142dd7efcb894e819e842693c9e1e17ff48872ff5","bin/ccemux.lua":"239476f58835b86bbcac31ce8af3c3acd3d198a55ab9ada78c62fbf358625a98","bin/chronometer.lua":"db5363993a04382145aef7db2fbe262f0bf10697a589e1e2d2f9ce0f87430dd8","bin/kristminer.lua":"7e7f9fe2a6493d584ad6926cda915e02c1c3d800dc209680898ce930d0bb0e6f","bin/livegps.lua":"c3d17d495cda01aa1261e4c4fcd43439b29af422671972117ec34f68e32c5bba","bin/loading.lua":"c85f7aa1765170325155b921c1fceeb62643f552f12d41b529a22af3a67f5a97","bin/potatoflight.lua":"2fbb0b6f8d78728d8cb0ec64af1bc598bd00cb55f202378e7acdb86bba71efd1","bin/potatoplex.lua":"86c9e7597bbe23d7de7e7f1bfc976d0b94dcdf3af9e6c7c6c9b18b98596898c8","bin/relay.lua":"261ae6c220b83506e3326e8f2b091d246baae458ff0d2ee87512be2c4e35a75d","bin/tryhaskell.lua":"07810d85145da65a3e434154c79d5a9d72f2dcbe59c8d6829040fb925df878ec","potatobios.lua":"7593e0f9d0755eea05e3c0fff5c4c9e036745a8cbc20e9e19609d8743a77b87e","signing-key.tbl":"b32af5229c23af3bc03d538e42751b26044e404a7b1af064ed89894efe421607","startup":"f17bfb9b4322c4467dc9170d50827f2d75717e5c3125d734f21f3406657917bc","update-key.hex":"8d8afb7a45833bb7d68f929421ad60a211d4d73e0ee03b24dc0106ba1de2e1a0","xlib/00_cbor.lua":"464b075e4f094b8db42506bd4bdaad0db87699ea7fbf80e5b87739b4aa9279af","xlib/01_skynet.lua":"9cb565d639a0acd7c763c3e7422482532cd0bda0cdfcc720089ab4a87e551339","xlib/03_heavlisp.lua":"dfdd5ccbaa8d3b761d9c517e4c8db8121a1ace508d4232af8f5a9cc9c876bcaf"},"timestamp":1611961247} -{"hash":"db5a274625ccd0defc6032bc84e49428f9ff405df4bbc125d5111a3bf8987704","sig":"792d3c585be37b7660e25d40381b422ecd7f08d216045b1bb2032547c2dc52ea176c243b7140f7cc8a21"} \ No newline at end of file +{"build":184,"description":"mild efficiency improvement on updates (fix even more so)","files":{"LICENSES":"f3549d84d66eb53dd4a421a4341d77d3d217c1b117d67e3be8f5211adcda0952","autorun.lua":"e8bcec7203fac0d544affd2ca1c23189f66b8648a99d9b5aaca4c418e981a045","bin/5rot26.lua":"91b66cd6d4b33081b25c456142dd7efcb894e819e842693c9e1e17ff48872ff5","bin/ccemux.lua":"239476f58835b86bbcac31ce8af3c3acd3d198a55ab9ada78c62fbf358625a98","bin/chronometer.lua":"db5363993a04382145aef7db2fbe262f0bf10697a589e1e2d2f9ce0f87430dd8","bin/kristminer.lua":"7e7f9fe2a6493d584ad6926cda915e02c1c3d800dc209680898ce930d0bb0e6f","bin/livegps.lua":"c3d17d495cda01aa1261e4c4fcd43439b29af422671972117ec34f68e32c5bba","bin/loading.lua":"c85f7aa1765170325155b921c1fceeb62643f552f12d41b529a22af3a67f5a97","bin/potatoflight.lua":"2fbb0b6f8d78728d8cb0ec64af1bc598bd00cb55f202378e7acdb86bba71efd1","bin/potatoplex.lua":"86c9e7597bbe23d7de7e7f1bfc976d0b94dcdf3af9e6c7c6c9b18b98596898c8","bin/relay.lua":"261ae6c220b83506e3326e8f2b091d246baae458ff0d2ee87512be2c4e35a75d","bin/tryhaskell.lua":"07810d85145da65a3e434154c79d5a9d72f2dcbe59c8d6829040fb925df878ec","potatobios.lua":"7593e0f9d0755eea05e3c0fff5c4c9e036745a8cbc20e9e19609d8743a77b87e","signing-key.tbl":"b32af5229c23af3bc03d538e42751b26044e404a7b1af064ed89894efe421607","startup":"f17bfb9b4322c4467dc9170d50827f2d75717e5c3125d734f21f3406657917bc","update-key.hex":"8d8afb7a45833bb7d68f929421ad60a211d4d73e0ee03b24dc0106ba1de2e1a0","xlib/00_cbor.lua":"464b075e4f094b8db42506bd4bdaad0db87699ea7fbf80e5b87739b4aa9279af","xlib/01_skynet.lua":"9cb565d639a0acd7c763c3e7422482532cd0bda0cdfcc720089ab4a87e551339","xlib/03_heavlisp.lua":"dfdd5ccbaa8d3b761d9c517e4c8db8121a1ace508d4232af8f5a9cc9c876bcaf"},"sizes":{"LICENSES":4725,"autorun.lua":182288,"bin/5rot26.lua":914,"bin/ccemux.lua":1673,"bin/chronometer.lua":1152,"bin/kristminer.lua":5566,"bin/livegps.lua":980,"bin/loading.lua":7707,"bin/potatoflight.lua":3417,"bin/potatoplex.lua":6526,"bin/relay.lua":3075,"bin/tryhaskell.lua":1867,"potatobios.lua":71272,"signing-key.tbl":190,"startup":8313,"update-key.hex":44,"xlib/00_cbor.lua":15808,"xlib/01_skynet.lua":3286,"xlib/03_heavlisp.lua":15399},"timestamp":1611962580} +{"hash":"998e6650ac5d772d3c594cb2bbdfb5d1bd6c166916dcfd70379146ce15c4e893","sig":"07110cf31af9f23d6da75999a4ec13a01d87709a27f6a31a656560e014761ed8aa43ef3f1a5d2ca6630a"} \ No newline at end of file diff --git a/src/main.lua b/src/main.lua index 5d73283..5a848bf 100644 --- a/src/main.lua +++ b/src/main.lua @@ -850,7 +850,8 @@ local function download_files(manifest_data, needed_files) local x = h.readAll() h.close() local hexsha = hexize(sha256(x)) - if manifest_data.files[file] ~= hexsha then error(("hash mismatch on %s %s (expected %s, got %s)"):format(file, url, manifest_data.files[file], hexsha)) end + if (manifest_data.sizes and manifest_data.sizes[file] and manifest_data.sizes[file] ~= #x) or manifest_data.files[file] ~= hexsha then + error(("hash mismatch on %s %s (expected %s, got %s)"):format(file, url, manifest_data.files[file], hexsha)) end fwrite(file, x) count = count + 1 end) @@ -911,7 +912,7 @@ local function process_manifest(url, force) for file, hash in pairs(data.files) do if fs.isDir(file) then fs.delete(file) end if not fs.exists(file) then print("missing", file) add_log("nonexistent %s", file) table.insert(needs, file) - elseif hexize(sha256(fread(file))) ~= hash then + elseif (data.sizes and data.sizes[file] and data.sizes[file] ~= fs.getSize(file)) or hexize(sha256(fread(file))) ~= hash then add_log("mismatch %s %s", file, hash) print("mismatch on", file, hash) table.insert(needs, file)