mirror of
https://github.com/osmarks/random-stuff
synced 2025-05-13 02:44:05 +00:00
number research
This commit is contained in:
osmarks
parent
6cadd09b4e
commit
d057781e3c
6
.gitignore
vendored
6
.gitignore
vendored
@ -8,4 +8,8 @@ code-guessing/analytics/people
|
||||
*.fls
|
||||
*.bbl
|
||||
*.out
|
||||
*.gz
|
||||
*.gz
|
||||
*/counts.csv
|
||||
*/counts-old.csv
|
||||
*/*.png
|
||||
*.zst
|
||||
|
||||
@ -13,7 +13,7 @@ TARGET_ID = 0x58
|
||||
CTRL_MEAS_REGISTER = 0xF4
|
||||
PRESSURE_REGISTER_BASE = 0xF7
|
||||
TEMP_REGISTER_BASE = 0xFA
|
||||
CALIBRAITON_VALUES = {
|
||||
CALIBRATION_VALUES = {
|
||||
"dig_T1": {"address": (0x88, 0x89), "unpack": "<H"},
|
||||
"dig_T2": {"address": (0x8A, 0x8B), "unpack": "<h"},
|
||||
"dig_T3": {"address": (0x8C, 0x8D), "unpack": "<h"},
|
||||
@ -32,7 +32,7 @@ def setup():
|
||||
assert bus.read_byte_data(ADDR, ID_REGISTER) == TARGET_ID
|
||||
bus.write_byte_data(ADDR, CTRL_MEAS_REGISTER, 0b101_101_11) # max oversampling mode (we don"t really care about power), power on
|
||||
calibration = {}
|
||||
for key, value in CALIBRAITON_VALUES.items():
|
||||
for key, value in CALIBRATION_VALUES.items():
|
||||
calibration[key] = struct.unpack(value["unpack"], bytes([bus.read_byte_data(ADDR, value["address"][0]), bus.read_byte_data(ADDR, value["address"][1])]))[0]
|
||||
return calibration
|
||||
|
||||
@ -81,4 +81,4 @@ start_http_server(9091)
|
||||
|
||||
while True:
|
||||
read_temperature_pressure()
|
||||
time.sleep(1)
|
||||
time.sleep(1)
|
||||
|
||||
594
empirical-number-distribution/Cargo.lock
generated
Normal file
594
empirical-number-distribution/Cargo.lock
generated
Normal file
@ -0,0 +1,594 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom 0.2.16",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.98"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
|
||||
|
||||
[[package]]
|
||||
name = "castaway"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"
|
||||
dependencies = [
|
||||
"castaway",
|
||||
"cfg-if",
|
||||
"itoa",
|
||||
"rustversion",
|
||||
"ryu",
|
||||
"serde",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||
|
||||
[[package]]
|
||||
name = "empirical-number-distribution"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"compact_str",
|
||||
"foldhash",
|
||||
"rayon",
|
||||
"sonic-rs",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "faststr"
|
||||
version = "0.2.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6503af7917fea18ffef8f7e8553fb8dff89e2e6837e94e09dd7fb069c82d62c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"rkyv",
|
||||
"serde",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasi 0.14.2+wasi-0.2.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
|
||||
dependencies = [
|
||||
"getrandom 0.3.2",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.172"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
|
||||
|
||||
[[package]]
|
||||
name = "munge"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e22e7961c873e8b305b176d2a4e1d41ce7ba31bc1c52d2a107a89568ec74c55"
|
||||
dependencies = [
|
||||
"munge_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "munge_macro"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ac7d860b767c6398e88fe93db73ce53eb496057aa6895ffa4d60cb02e1d1c6b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptr_meta"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90"
|
||||
dependencies = [
|
||||
"ptr_meta_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptr_meta_derive"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
|
||||
|
||||
[[package]]
|
||||
name = "rancor"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947"
|
||||
dependencies = [
|
||||
"ptr_meta",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rend"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a35e8a6bf28cd121053a66aa2e6a2e3eaffad4a60012179f0e864aa5ffeff215"
|
||||
|
||||
[[package]]
|
||||
name = "rkyv"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e147371c75553e1e2fcdb483944a8540b8438c31426279553b9a8182a9b7b65"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hashbrown",
|
||||
"indexmap",
|
||||
"munge",
|
||||
"ptr_meta",
|
||||
"rancor",
|
||||
"rend",
|
||||
"rkyv_derive",
|
||||
"tinyvec",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rkyv_derive"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "246b40ac189af6c675d124b802e8ef6d5246c53e17367ce9501f8f66a81abb7a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.219"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.219"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
|
||||
|
||||
[[package]]
|
||||
name = "sonic-number"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8a74044c092f4f43ca7a6cfd62854cf9fb5ac8502b131347c990bf22bef1dfe"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonic-rs"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93070f7e7c0d7ec7d08406b1b407234af30420320fd854f304029e3c6db4a899"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bumpalo",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"faststr",
|
||||
"itoa",
|
||||
"ref-cast",
|
||||
"ryu",
|
||||
"serde",
|
||||
"simdutf8",
|
||||
"sonic-number",
|
||||
"sonic-simd",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonic-simd"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b421f7b6aa4a5de8f685aaf398dfaa828346ee639d2b1c1061ab43d40baa6223"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.101"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.14.2+wasi-0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
|
||||
dependencies = [
|
||||
"wit-bindgen-rt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rt"
|
||||
version = "0.39.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.0.15+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
12
empirical-number-distribution/Cargo.toml
Normal file
12
empirical-number-distribution/Cargo.toml
Normal file
@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "empirical-number-distribution"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.98"
|
||||
compact_str = { version = "0.9.0", features = ["serde"] }
|
||||
foldhash = "0.1.5"
|
||||
rayon = "1.10.0"
|
||||
sonic-rs = "0.5.1"
|
||||
zstd = "0.13.3"
|
||||
151
empirical-number-distribution/analysis.py
Normal file
151
empirical-number-distribution/analysis.py
Normal file
@ -0,0 +1,151 @@
|
||||
import polars as pl
|
||||
import numpy as np
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import math
|
||||
df = pl.read_csv("counts.csv", schema={"number": pl.String, "count": pl.Int64})
|
||||
|
||||
def compute_zipflike(df, k):
|
||||
topk = df.top_k(k, by=df["count"])
|
||||
frequencies = topk[:, 1].to_numpy()
|
||||
ranks = np.arange(len(frequencies)) + 1
|
||||
|
||||
log_frequencies = np.log(frequencies)
|
||||
log_ranks = np.log(ranks)
|
||||
|
||||
# https://numpy.org/doc/stable/reference/generated/numpy.linalg.lstsq.html
|
||||
A = np.vstack([log_ranks, np.ones(len(log_ranks))]).T
|
||||
gradient, y_intercept = np.linalg.lstsq(A, log_frequencies)[0]
|
||||
|
||||
predicted_log_frequencies = log_ranks * gradient + y_intercept
|
||||
|
||||
predicted_log_frequencies_zipf_gradient = log_ranks * -1.0
|
||||
rms_y_intercept_zipf = np.sqrt(np.mean((predicted_log_frequencies_zipf_gradient - log_frequencies) ** 2))
|
||||
predicted_log_frequencies_zipf_gradient = log_ranks * -1.0 + rms_y_intercept_zipf
|
||||
|
||||
plt.title(f"Top {k} numbers")
|
||||
plt.xlabel("log(rank)")
|
||||
plt.ylabel("log(frequency)")
|
||||
plt.scatter(log_ranks, log_frequencies, label="empirical", color="blue")
|
||||
plt.plot(log_ranks, predicted_log_frequencies, label=f"lstsq fit gradient={gradient:.2f}", color="lime")
|
||||
plt.plot(log_ranks, predicted_log_frequencies_zipf_gradient, label=f"lstsq fit zipf", color="red")
|
||||
plt.legend()
|
||||
plt.tight_layout()
|
||||
plt.savefig(f"top_{k}_numbers.png")
|
||||
#plt.show()
|
||||
plt.close()
|
||||
|
||||
def compact_cat(x):
|
||||
st, en = json.loads(x.replace("(", "["))
|
||||
return f"{st:.0e}-{en:.0e}"
|
||||
|
||||
def strings_to_numbers(df):
|
||||
is_percent = df[:, 0].str.ends_with("%")
|
||||
stripped = df[:, 0].str.strip_suffix("%")
|
||||
scale = pl.when(is_percent).then(0.01).otherwise(1)
|
||||
numbers = stripped.cast(pl.Float64, strict=False)
|
||||
return df.with_columns(numbers * scale, df[:, 1])
|
||||
|
||||
def frequency_plot_for(values, counts, name, xs, scale="log", ticks=None, axline=None, xlim=None):
|
||||
plt.title("Number frequencies")
|
||||
ys = [ counts[values == x].sum() for x in xs ]
|
||||
plt.plot(xs, ys)
|
||||
plt.ylabel("count")
|
||||
plt.xlabel("number")
|
||||
plt.yscale(scale)
|
||||
if ticks:
|
||||
plt.xticks(ticks, minor=True)
|
||||
if axline:
|
||||
plt.axvline(axline, color="red")
|
||||
if xlim:
|
||||
plt.xlim(xlim)
|
||||
plt.savefig(f"{name}_frequency.png")
|
||||
plt.close()
|
||||
|
||||
with pl.Config() as cfg:
|
||||
cfg.set_tbl_formatting("ASCII_MARKDOWN")
|
||||
cfg.set_tbl_rows(100)
|
||||
cfg.set_tbl_hide_column_data_types(True)
|
||||
|
||||
print("len")
|
||||
print(len(df))
|
||||
|
||||
print("total count")
|
||||
total_count = df[:, 1].sum()
|
||||
print(total_count)
|
||||
|
||||
print("top 30")
|
||||
print(df.top_k(30, by=df["count"]))
|
||||
|
||||
print("frequency/rank")
|
||||
compute_zipflike(df, 1_000)
|
||||
compute_zipflike(df, 10_000)
|
||||
compute_zipflike(df, 100_000)
|
||||
|
||||
print("histogram")
|
||||
cats, counts = df[:, 1].hist(bins=np.geomspace(1, max(df[:, 1]), num=20), include_category=True, include_breakpoint=False)
|
||||
fig, ax = plt.subplots()
|
||||
plt.title("Frequency of number frequencies")
|
||||
ax.set_yscale("log")
|
||||
plt.xticks(rotation=45, ha="right")
|
||||
ax.bar([ compact_cat(x) for x in cats ], counts.to_numpy())
|
||||
#plt.show()
|
||||
fig.subplots_adjust(bottom=0.2)
|
||||
plt.savefig("number_freq_freq.png")
|
||||
plt.close()
|
||||
|
||||
print("benford")
|
||||
real_counts = {}
|
||||
real_counts_frac = {}
|
||||
benford_frequencies = {}
|
||||
for first_digit in range(1, 10):
|
||||
first_digit_s = str(first_digit)
|
||||
bcount = df.filter(df[:, 0].str.starts_with(first_digit_s))[:, 1].sum()
|
||||
bcount_frac = df.filter(df[:, 0].str.starts_with(first_digit_s) & (df[:, 0].str.contains(".", literal=True)))[:, 1].sum()
|
||||
print(bcount, bcount_frac)
|
||||
real_counts[first_digit_s] = bcount
|
||||
real_counts_frac[first_digit_s] = bcount_frac
|
||||
benford_frequencies[first_digit_s] = math.log10(first_digit + 1) - math.log10(first_digit)
|
||||
total_dcount = sum(real_counts.values())
|
||||
total_dfcount = sum(real_counts_frac.values())
|
||||
for k in real_counts:
|
||||
real_counts[k] /= total_dcount
|
||||
real_counts_frac[k] /= total_dfcount
|
||||
print(real_counts, real_counts_frac)
|
||||
plt.plot(list(real_counts.keys()), list(real_counts.values()), label="Empirical")
|
||||
plt.plot(list(real_counts_frac.keys()), list(real_counts_frac.values()), label="Empirical (noninteger)")
|
||||
plt.plot(list(benford_frequencies.keys()), list(benford_frequencies.values()), label="Benford")
|
||||
plt.xlabel("First digit")
|
||||
plt.ylabel("Frequency (relative)")
|
||||
plt.legend()
|
||||
plt.savefig("benford.png")
|
||||
plt.close()
|
||||
|
||||
print("to float domain")
|
||||
numbers = strings_to_numbers(df)
|
||||
numbers_n = numbers[:, 0].to_numpy()
|
||||
numbers_c = numbers[:, 1].to_numpy()
|
||||
|
||||
print("median number")
|
||||
perm = np.argsort(numbers_n)
|
||||
ccounts = numbers_c[perm].cumsum()
|
||||
midpoint = total_count // 2
|
||||
midpoint_index = np.searchsorted(ccounts, midpoint)
|
||||
print(numbers_n[perm[midpoint_index]])
|
||||
|
||||
log_numbers = np.log(np.abs(numbers_n))
|
||||
log_numbers = log_numbers[np.isfinite(log_numbers)]
|
||||
|
||||
print("number size histogram")
|
||||
counts, bins = np.histogram(log_numbers, bins=256)
|
||||
plt.title("Number sizes histogram")
|
||||
plt.stairs(counts, bins)
|
||||
plt.yscale("log")
|
||||
plt.axvline(0)
|
||||
plt.ylabel("density")
|
||||
plt.xlabel("log(number)")
|
||||
plt.savefig("number_size_histogram.png")
|
||||
plt.close()
|
||||
|
||||
frequency_plot_for(numbers_n, numbers_c, "small_numbers", np.arange(100), ticks=[ n for n in range(0, 100, 10) ] + [ 2**n for n in range(0, 7) ])
|
||||
frequency_plot_for(numbers_n, numbers_c, "years", np.arange(1900, 2100), scale="linear", axline=2020, ticks=[ n for n in range(1900, 2100, 10) ], xlim=0)
|
||||
215
empirical-number-distribution/src/main.rs
Normal file
215
empirical-number-distribution/src/main.rs
Normal file
@ -0,0 +1,215 @@
|
||||
use anyhow::{Context, Result};
|
||||
use compact_str::CompactString;
|
||||
use std::{fs, io::{BufRead, BufReader, BufWriter, Write}, path::PathBuf, sync::{Arc, Mutex}};
|
||||
use sonic_rs::JsonValueTrait;
|
||||
use foldhash::{HashMap, HashMapExt};
|
||||
use rayon::prelude::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScannerState {
|
||||
may_start_number: bool,
|
||||
current_number_start: Option<usize>,
|
||||
number_has_digits: bool,
|
||||
was_hyphen: bool,
|
||||
was_alpha: bool,
|
||||
multiple_hyphens: bool
|
||||
}
|
||||
|
||||
fn extract_numbers<'a, F: FnMut(&'a str)>(s: &'a str, mut callback: F) {
|
||||
let mut state = ScannerState {
|
||||
may_start_number: true,
|
||||
current_number_start: None,
|
||||
number_has_digits: false,
|
||||
was_hyphen: false,
|
||||
was_alpha: false,
|
||||
multiple_hyphens: false
|
||||
};
|
||||
|
||||
let mut commit_segment = |segment: &'a str| {
|
||||
let dot_count = segment.chars().filter(|x| *x == '.').count();
|
||||
if dot_count > 1 { return; }
|
||||
let buffer = segment.strip_suffix(".").unwrap_or(segment);
|
||||
if buffer.len() > 0 {
|
||||
callback(buffer);
|
||||
}
|
||||
};
|
||||
|
||||
let mut commit = |pos, state: &mut ScannerState| {
|
||||
if let Some(start) = state.current_number_start {
|
||||
if state.number_has_digits {
|
||||
let chunk = &s[start..pos];
|
||||
let mut interpret_as_contiguous = true;
|
||||
// postprocessing
|
||||
for (i, seg) in chunk.split(",").enumerate() {
|
||||
if i > 0 && seg.len() != 3 && seg.len() > 0 {
|
||||
interpret_as_contiguous = false;
|
||||
}
|
||||
}
|
||||
if interpret_as_contiguous {
|
||||
commit_segment(chunk.strip_suffix(",").unwrap_or(chunk));
|
||||
} else {
|
||||
for seg in chunk.split(",") {
|
||||
commit_segment(seg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
state.current_number_start = None;
|
||||
state.may_start_number = true;
|
||||
state.number_has_digits = false;
|
||||
};
|
||||
|
||||
for (i, c) in s.char_indices() {
|
||||
//println!("{:?} {:?}", c, state);
|
||||
if c == '-' {
|
||||
state.multiple_hyphens = state.was_hyphen;
|
||||
} else {
|
||||
state.multiple_hyphens = false;
|
||||
}
|
||||
if state.may_start_number {
|
||||
if c.is_ascii_digit() {
|
||||
state.current_number_start = Some(i);
|
||||
state.number_has_digits = true;
|
||||
state.may_start_number = false;
|
||||
}
|
||||
if c == '-' && !state.was_hyphen && !state.was_alpha {
|
||||
state.current_number_start = Some(i);
|
||||
state.number_has_digits = false;
|
||||
state.may_start_number = false;
|
||||
state.was_hyphen = true;
|
||||
}
|
||||
}
|
||||
match c {
|
||||
c if c.is_ascii_digit() => {
|
||||
state.number_has_digits = true;
|
||||
},
|
||||
'%' => commit(i+1, &mut state),
|
||||
'-' => {
|
||||
if !state.was_hyphen || state.number_has_digits || state.multiple_hyphens { commit(i, &mut state); }
|
||||
state.was_hyphen = true;
|
||||
},
|
||||
'–' => commit(i, &mut state),
|
||||
',' => (),
|
||||
'.' => (),
|
||||
_ => {
|
||||
commit(i, &mut state);
|
||||
if c.is_whitespace() { state.was_hyphen = false; }
|
||||
}
|
||||
}
|
||||
state.was_alpha = c.is_alphabetic();
|
||||
}
|
||||
|
||||
commit(s.len(), &mut state);
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let root = PathBuf::from(args.get(1).context("root path required")?);
|
||||
let mut numbers: Arc<Mutex<HashMap<CompactString, u64>>> = Arc::new(Mutex::new(HashMap::new()));
|
||||
let mut paths = vec![];
|
||||
|
||||
for target in root.read_dir()? {
|
||||
let target = target?;
|
||||
if target.file_name().to_str().context("non-UTF8 filepath (no)")?.ends_with(".jsonl.zst") {
|
||||
paths.push(target.path());
|
||||
}
|
||||
}
|
||||
|
||||
paths.into_par_iter().try_for_each_with(numbers.clone(), |global_numbers, path| -> Result<()> {
|
||||
let mut numbers: HashMap<CompactString, u64> = HashMap::new();
|
||||
|
||||
let file = BufReader::with_capacity(32*1024*1024, fs::File::open(path)?);
|
||||
let mut decoder = BufReader::new(zstd::Decoder::new(file)?);
|
||||
let mut input = String::with_capacity(8192);
|
||||
loop {
|
||||
if decoder.read_line(&mut input)? == 0 {
|
||||
break;
|
||||
}
|
||||
let text = sonic_rs::get_from_str(&input, &["text"])?;
|
||||
let text = text.as_str().context("invalid format")?;
|
||||
//println!("{:?}", text);
|
||||
extract_numbers(text, |num| {
|
||||
let key = num.replace(",", "");
|
||||
if key == "%" {
|
||||
return;
|
||||
}
|
||||
let has_percent = key.ends_with("%");
|
||||
let has_minus = key.starts_with("-");
|
||||
let key = key.trim_start_matches("0").trim_end_matches("%").trim_start_matches("-");
|
||||
let mut key = if key.starts_with(".") || key.is_empty() {
|
||||
let mut new_key = CompactString::new("0");
|
||||
new_key.push_str(key);
|
||||
new_key
|
||||
} else {
|
||||
CompactString::from(key)
|
||||
};
|
||||
|
||||
if let Some(decimal_point) = key.find('.') {
|
||||
key.truncate(decimal_point + &key[decimal_point..key.len()].trim_end_matches("0").len());
|
||||
if key.ends_with(".") {
|
||||
key.truncate(key.len() - 1);
|
||||
}
|
||||
if key.len() == 0 {
|
||||
key.push_str("0");
|
||||
}
|
||||
}
|
||||
|
||||
if has_minus {
|
||||
key.insert(0, '-');
|
||||
}
|
||||
|
||||
if has_percent {
|
||||
key.push_str("%");
|
||||
}
|
||||
|
||||
*numbers.entry(key).or_default() += 1;
|
||||
});
|
||||
input.clear();
|
||||
}
|
||||
|
||||
{
|
||||
let mut global_numbers = global_numbers.lock().unwrap();
|
||||
for (key, count) in numbers {
|
||||
*global_numbers.entry(key).or_default() += count;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
let mut out_file = BufWriter::new(fs::File::create(args.get(2).context("output path required")?)?);
|
||||
|
||||
write!(&mut out_file, "number,count\n")?;
|
||||
for (key, count) in std::mem::replace(Arc::get_mut(&mut numbers).unwrap(), Mutex::new(HashMap::new())).into_inner().unwrap() {
|
||||
write!(&mut out_file, "{},{}\n", key, count)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex() {
|
||||
let test_cases = &[
|
||||
("106 bees approach 45675 nonbees", vec!["106", "45675"]),
|
||||
("2, then fewer, then -5", vec!["2", "-5"]),
|
||||
("1--3 is a weird thing to write", vec!["1", "3"]),
|
||||
("version 4.6.0 of the software", vec![]),
|
||||
("it has been shown that 841% of users, or up to 25, liked the new version", vec!["841%", "25"]),
|
||||
("by then, 1,401,041 numbers had been found by 6,436 people and -44,036,110", vec!["1,401,041", "6,436", "-44,036,110"]),
|
||||
("translate(0,0,0)", vec!["0", "0", "0"]),
|
||||
("in this case, 10-30 means ten to thirty and 10–30 (en dash) also means that", vec!["10", "30", "10", "30"]),
|
||||
("issued Dec. 21, 1999, and U.S. Pat. No. 5,777,999, entitled", vec!["21", "1999", "5,777,999"]),
|
||||
("Forty days after Farkhunda, a 27-year-old Afghan woman falsely accused of burning a copy of the Quran, was publicly beaten and burnt to death on 1, March 19, 2015,", vec!["27", "1", "19", "2015"]),
|
||||
("4.6 million integers, -0 negative zeroes, 1,2,3", vec!["4.6", "-0", "1", "2", "3"]),
|
||||
("move -20px and consume $30 at once, then 11,22,33", vec!["-20", "30", "11", "22", "33"]),
|
||||
("I used up a DC-8 doing my W-2 form", vec!["8", "2"]),
|
||||
("[[@b11-kjim-2015-406],[@b20-kjim-2015-406]\\]", vec!["11", "2015", "406", "20", "2015", "406"]),
|
||||
("--385.12 and ---411 and 0", vec!["385.12", "411", "0"])
|
||||
];
|
||||
|
||||
for (input, output) in test_cases {
|
||||
let mut matches: Vec<&str> = vec![];
|
||||
extract_numbers(input, |mat| matches.push(mat));
|
||||
assert_eq!(matches.as_slice(), output.as_slice());
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user