6_4/potatOS
1
0
Fork 0
forked from osmarks/potatOS
Code Pull Requests Activity

slightly faster invalidation via size tracking

Browse Source
This commit is contained in:
osmarks 2021-01-29 23:25:09 +00:00
parent e0781c1f94
commit 7424beaa2d
3 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
generate_manifest.py
View File

@ -22,8 +22,11 @@ if os.path.exists("./manifest"):
def hash_file(path): def hash_file(path):
file = open(path, "rb") file = open(path, "rb")
h = hashlib.sha256() h = hashlib.sha256()
while data := file.read(65536): h.update(data) count = 0
return h.hexdigest() while data := file.read(65536):
h.update(data)
count += len(data)
return h.hexdigest(), count
if args.sign: if args.sign:
print("Signing update") print("Signing update")
@ -33,16 +36,21 @@ if args.sign:
open("dist/update-key.hex", "w").write(pubkey) open("dist/update-key.hex", "w").write(pubkey)
files = dict() files = dict()
sizes = dict()
code = Path("./dist/") code = Path("./dist/")
for path in code.glob("**/*"): for path in code.glob("**/*"):
if not path.is_dir(): if not path.is_dir():
files["/".join(path.parts[1:])] = hash_file(path) hexhash, count = hash_file(path)
mpath = "/".join(path.parts[1:])
files[mpath] = hexhash
sizes[mpath] = count
def deterministic_json_serialize(x): def deterministic_json_serialize(x):
return json.dumps(x, sort_keys=True, separators=(",", ":")) return json.dumps(x, sort_keys=True, separators=(",", ":"))
manifest_data = deterministic_json_serialize({ manifest_data = deterministic_json_serialize({
"files": files, "files": files,
"sizes": sizes,
"timestamp": int(datetime.datetime.now().timestamp()), "timestamp": int(datetime.datetime.now().timestamp()),
"build": counter + 1, "build": counter + 1,
"description": args.description "description": args.description

4
manifest
View File

@ -1,2 +1,2 @@
{"build":180,"description":"fix info","files":{"LICENSES":"f3549d84d66eb53dd4a421a4341d77d3d217c1b117d67e3be8f5211adcda0952","autorun.lua":"9392777418bbd861836a95c0df6b254a914d861c4d7d73baa48867e4f7f2a8e9","bin/5rot26.lua":"91b66cd6d4b33081b25c456142dd7efcb894e819e842693c9e1e17ff48872ff5","bin/ccemux.lua":"239476f58835b86bbcac31ce8af3c3acd3d198a55ab9ada78c62fbf358625a98","bin/chronometer.lua":"db5363993a04382145aef7db2fbe262f0bf10697a589e1e2d2f9ce0f87430dd8","bin/kristminer.lua":"7e7f9fe2a6493d584ad6926cda915e02c1c3d800dc209680898ce930d0bb0e6f","bin/livegps.lua":"c3d17d495cda01aa1261e4c4fcd43439b29af422671972117ec34f68e32c5bba","bin/loading.lua":"c85f7aa1765170325155b921c1fceeb62643f552f12d41b529a22af3a67f5a97","bin/potatoflight.lua":"2fbb0b6f8d78728d8cb0ec64af1bc598bd00cb55f202378e7acdb86bba71efd1","bin/potatoplex.lua":"86c9e7597bbe23d7de7e7f1bfc976d0b94dcdf3af9e6c7c6c9b18b98596898c8","bin/relay.lua":"261ae6c220b83506e3326e8f2b091d246baae458ff0d2ee87512be2c4e35a75d","bin/tryhaskell.lua":"07810d85145da65a3e434154c79d5a9d72f2dcbe59c8d6829040fb925df878ec","potatobios.lua":"7593e0f9d0755eea05e3c0fff5c4c9e036745a8cbc20e9e19609d8743a77b87e","signing-key.tbl":"b32af5229c23af3bc03d538e42751b26044e404a7b1af064ed89894efe421607","startup":"f17bfb9b4322c4467dc9170d50827f2d75717e5c3125d734f21f3406657917bc","update-key.hex":"8d8afb7a45833bb7d68f929421ad60a211d4d73e0ee03b24dc0106ba1de2e1a0","xlib/00_cbor.lua":"464b075e4f094b8db42506bd4bdaad0db87699ea7fbf80e5b87739b4aa9279af","xlib/01_skynet.lua":"9cb565d639a0acd7c763c3e7422482532cd0bda0cdfcc720089ab4a87e551339","xlib/03_heavlisp.lua":"dfdd5ccbaa8d3b761d9c517e4c8db8121a1ace508d4232af8f5a9cc9c876bcaf"},"timestamp":1611961247} {"build":184,"description":"mild efficiency improvement on updates (fix even more so)","files":{"LICENSES":"f3549d84d66eb53dd4a421a4341d77d3d217c1b117d67e3be8f5211adcda0952","autorun.lua":"e8bcec7203fac0d544affd2ca1c23189f66b8648a99d9b5aaca4c418e981a045","bin/5rot26.lua":"91b66cd6d4b33081b25c456142dd7efcb894e819e842693c9e1e17ff48872ff5","bin/ccemux.lua":"239476f58835b86bbcac31ce8af3c3acd3d198a55ab9ada78c62fbf358625a98","bin/chronometer.lua":"db5363993a04382145aef7db2fbe262f0bf10697a589e1e2d2f9ce0f87430dd8","bin/kristminer.lua":"7e7f9fe2a6493d584ad6926cda915e02c1c3d800dc209680898ce930d0bb0e6f","bin/livegps.lua":"c3d17d495cda01aa1261e4c4fcd43439b29af422671972117ec34f68e32c5bba","bin/loading.lua":"c85f7aa1765170325155b921c1fceeb62643f552f12d41b529a22af3a67f5a97","bin/potatoflight.lua":"2fbb0b6f8d78728d8cb0ec64af1bc598bd00cb55f202378e7acdb86bba71efd1","bin/potatoplex.lua":"86c9e7597bbe23d7de7e7f1bfc976d0b94dcdf3af9e6c7c6c9b18b98596898c8","bin/relay.lua":"261ae6c220b83506e3326e8f2b091d246baae458ff0d2ee87512be2c4e35a75d","bin/tryhaskell.lua":"07810d85145da65a3e434154c79d5a9d72f2dcbe59c8d6829040fb925df878ec","potatobios.lua":"7593e0f9d0755eea05e3c0fff5c4c9e036745a8cbc20e9e19609d8743a77b87e","signing-key.tbl":"b32af5229c23af3bc03d538e42751b26044e404a7b1af064ed89894efe421607","startup":"f17bfb9b4322c4467dc9170d50827f2d75717e5c3125d734f21f3406657917bc","update-key.hex":"8d8afb7a45833bb7d68f929421ad60a211d4d73e0ee03b24dc0106ba1de2e1a0","xlib/00_cbor.lua":"464b075e4f094b8db42506bd4bdaad0db87699ea7fbf80e5b87739b4aa9279af","xlib/01_skynet.lua":"9cb565d639a0acd7c763c3e7422482532cd0bda0cdfcc720089ab4a87e551339","xlib/03_heavlisp.lua":"dfdd5ccbaa8d3b761d9c517e4c8db8121a1ace508d4232af8f5a9cc9c876bcaf"},"sizes":{"LICENSES":4725,"autorun.lua":182288,"bin/5rot26.lua":914,"bin/ccemux.lua":1673,"bin/chronometer.lua":1152,"bin/kristminer.lua":5566,"bin/livegps.lua":980,"bin/loading.lua":7707,"bin/potatoflight.lua":3417,"bin/potatoplex.lua":6526,"bin/relay.lua":3075,"bin/tryhaskell.lua":1867,"potatobios.lua":71272,"signing-key.tbl":190,"startup":8313,"update-key.hex":44,"xlib/00_cbor.lua":15808,"xlib/01_skynet.lua":3286,"xlib/03_heavlisp.lua":15399},"timestamp":1611962580}
{"hash":"db5a274625ccd0defc6032bc84e49428f9ff405df4bbc125d5111a3bf8987704","sig":"792d3c585be37b7660e25d40381b422ecd7f08d216045b1bb2032547c2dc52ea176c243b7140f7cc8a21"} {"hash":"998e6650ac5d772d3c594cb2bbdfb5d1bd6c166916dcfd70379146ce15c4e893","sig":"07110cf31af9f23d6da75999a4ec13a01d87709a27f6a31a656560e014761ed8aa43ef3f1a5d2ca6630a"}

5
src/main.lua
View File

@ -850,7 +850,8 @@ local function download_files(manifest_data, needed_files)
local x = h.readAll() local x = h.readAll()
h.close() h.close()
local hexsha = hexize(sha256(x)) local hexsha = hexize(sha256(x))
if manifest_data.files[file] ~= hexsha then error(("hash mismatch on %s %s (expected %s, got %s)"):format(file, url, manifest_data.files[file], hexsha)) end if (manifest_data.sizes and manifest_data.sizes[file] and manifest_data.sizes[file] ~= #x) or manifest_data.files[file] ~= hexsha then
error(("hash mismatch on %s %s (expected %s, got %s)"):format(file, url, manifest_data.files[file], hexsha)) end
fwrite(file, x) fwrite(file, x)
count = count + 1 count = count + 1
end) end)
@ -911,7 +912,7 @@ local function process_manifest(url, force)
for file, hash in pairs(data.files) do for file, hash in pairs(data.files) do
if fs.isDir(file) then fs.delete(file) end if fs.isDir(file) then fs.delete(file) end
if not fs.exists(file) then print("missing", file) add_log("nonexistent %s", file) table.insert(needs, file) if not fs.exists(file) then print("missing", file) add_log("nonexistent %s", file) table.insert(needs, file)
elseif hexize(sha256(fread(file))) ~= hash then elseif (data.sizes and data.sizes[file] and data.sizes[file] ~= fs.getSize(file)) or hexize(sha256(fread(file))) ~= hash then
add_log("mismatch %s %s", file, hash) add_log("mismatch %s %s", file, hash)
print("mismatch on", file, hash) print("mismatch on", file, hash)
table.insert(needs, file) table.insert(needs, file)