From ee23b8144405c5af44e4302a468e0caa4d64e37f Mon Sep 17 00:00:00 2001 From: osmarks Date: Fri, 24 Jan 2025 09:24:28 +0000 Subject: [PATCH] release version --- .gitignore | 5 + Cargo.lock | 41 +++++ Cargo.toml | 2 + clipfront2/src/About.svelte | 8 +- clipfront2/src/App.svelte | 29 +++- clipfront2/src/QueryRefiner.svelte | 20 +-- clipfront2/src/SearchResults.svelte | 8 +- clipfront2/src/common.sass | 7 + clipfront2/src/iosevka.woff2 | Bin 0 -> 13844 bytes clipfront2/src/util.js | 12 ++ clipfront2/static/index.html | 2 +- config2.json | 10 ++ diskann/chainq.py | 164 ++++++++++++++++++ diskann/src/vector.rs | 2 +- faiss_bench_quantizer.py | 131 +++++++++++++++ frontend_config.json | 8 +- genseahash.py | 4 + meme-rater/active_learning.py | 3 +- meme-rater/active_learning_find_top.py | 2 +- meme-rater/active_learning_gradients.py | 11 +- meme-rater/ensemble_to_wide_model.py | 16 +- meme-rater/model.py | 2 +- meme-rater/rater_server.py | 54 +++--- meme-rater/shared.py | 18 +- meme-rater/train.py | 7 +- sae/shared.py | 8 +- sae/train.py | 11 +- slow_dump_parse_script.py | 35 ++++ src/common.rs | 13 +- src/dump_processor.rs | 57 +++++-- src/get_embedding.py | 12 +- src/main.rs | 5 +- src/query_disk_index.rs | 210 ++++++++++++++++++++---- src/score_model.rs | 4 +- 34 files changed, 774 insertions(+), 147 deletions(-) create mode 100644 clipfront2/src/iosevka.woff2 create mode 100644 config2.json create mode 100644 diskann/chainq.py create mode 100644 faiss_bench_quantizer.py create mode 100644 genseahash.py create mode 100644 slow_dump_parse_script.py diff --git a/.gitignore b/.gitignore index eca5fa3..9df9395 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ index queries.txt *.zst .safetensors +*/static/*.woff2 +flamegraph.svg +*.jsonl +*.safetensors +perf.data diff --git a/Cargo.lock b/Cargo.lock index 846ce22..cd7d312 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,6 +416,18 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b81e1519b0d82120d2fd469d5bfb2919a9361c48b02d82d04befc1cdd2002452" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -1117,6 +1129,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.31" @@ -2046,6 +2064,7 @@ dependencies = [ "axum", "base64 0.22.1", "bitcode", + "bitvec", "bytemuck", "candle-core", "chrono", @@ -2067,6 +2086,7 @@ dependencies = [ "itertools 0.13.0", "json5", "lazy_static", + "matrixmultiply", "maud", "memmap2", "mimalloc", @@ -2850,6 +2870,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -3873,6 +3899,12 @@ dependencies = [ "version-compare", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.12.16" @@ -4732,6 +4764,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index ee9357b..39bd6e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ monoio = "0.2" hyper = "1" monoio-compat = { version = "0.2", features = ["hyper"] } http-body-util = "0.1" +matrixmultiply = "0.3" +bitvec = "1" [[bin]] name = "reddit-dump" diff --git a/clipfront2/src/About.svelte b/clipfront2/src/About.svelte index 2b61a4c..f1ba35e 100644 --- a/clipfront2/src/About.svelte +++ b/clipfront2/src/About.svelte @@ -9,15 +9,15 @@

- Welcome to {util.hardConfig.name} by osmarks.net Computational Memetics. {util.hardConfig.name} searches images using semantic image/text embedding models. In general, search by thinking of what caption your desired image might have been given by random people on the internet. The model currently in use can read text fairly well and understands moderately abstract properties of images, but is limited to English and case-insensitive. + Welcome to {util.hardConfig.name} by osmarks.net Computational Memetics. {util.hardConfig.name} searches images using semantic image/text embedding models (refer to https://arxiv.org/abs/2303.15343, https://arxiv.org/abs/2103.00020). In general, search by thinking of what caption your desired image might have been given by random people on the internet. The model currently in use can read text fairly well and understands moderately abstract properties of images, but is limited to English and case-insensitive.

- Advanced Mode sliders are generated from PCA on the index. The human-readable labels are generated manually by looking at things. + "Useful"/"aesthetic"/"meme" sliders are defined based on an approximate extrapolation of my preferences and may not agree with your own opinions. Results are otherwise decided by the inscrutable whims of a billion-parameter neural network. Please note that large-scale internet data may contain things you do not like.

- The code is open-source and available on GitHub. + The code is open-source and available on GitHub.

- {#if util.hardConfig.telemetryEndpoint} + {#if util.hardConfig.telemetry_endpoint}

Privacy

We do not collect personal information. We do collect usage information (associated with a random ID) to improve the ranking algorithms. You can disable this: diff --git a/clipfront2/src/App.svelte b/clipfront2/src/App.svelte index b0a7bef..ed5b896 100644 --- a/clipfront2/src/App.svelte +++ b/clipfront2/src/App.svelte @@ -154,7 +154,6 @@

Advanced About - Refine
@@ -185,7 +184,11 @@ - + {#if debugEnabled} + + {:else} + + {/if} {#if term.type === "image"} {term.file.name} {:else if term.type === "text"} @@ -200,6 +203,14 @@ {/each} + {#if showDebugSwitch} +
+ + + {#if debugEnabled} + {/if} +
+ {/if}
@@ -242,6 +253,9 @@ let queryCounter = 0 let config = {} + const showDebugSwitch = localStorage.getItem("debugEnabled") === "true" + let debugEnabled = false + const newTextQuery = (content=null) => { queryTerms.push({ type: "text", weight: 1, sign: "+", text: typeof content === "string" ? content : "" }) queryTerms = queryTerms @@ -253,9 +267,14 @@ const runSearch = async () => { if (!resultPromise) { + const friendlyModeQueryOrRandom = friendlyModeQuery ? [{ text: friendlyModeQuery, weight: 1, sign: "+" }] : [{ embedding: util.randn(config.d_emb, 1 / (config.d_emb ** 0.5)), weight: 1, sign: "+" }] + const terms = friendlyMode ? + friendlyModeQueryOrRandom.concat(util.hardConfig.friendly_mode_default_terms ?? []) : + queryTerms.filter(x => x.text !== "").map(x => ({ image: x.imageData, text: x.text, embedding: x.embedding, predefined_embedding: x.predefinedEmbedding, weight: x.weight * { "+": 1, "-": -1 }[x.sign] })) let args = { - "terms": friendlyMode ? [{ text: friendlyModeQuery, weight: 1, sign: "+" }] : queryTerms.filter(x => x.text !== "").map(x => ({ image: x.imageData, text: x.text, embedding: x.embedding, predefined_embedding: x.predefinedEmbedding, weight: x.weight * { "+": 1, "-": -1 }[x.sign] })), - "include_video": true + "terms": terms, + "include_video": true, + "debug_enabled": debugEnabled } util.sendTelemetry("search", { @@ -314,7 +333,7 @@ type: "predefined_embedding", predefinedEmbedding: predefinedEmbeddingName, sign: "+", - weight: 0.2 + weight: 1 }) } queryTerms = queryTerms diff --git a/clipfront2/src/QueryRefiner.svelte b/clipfront2/src/QueryRefiner.svelte index 5055d8d..49e2dcf 100644 --- a/clipfront2/src/QueryRefiner.svelte +++ b/clipfront2/src/QueryRefiner.svelte @@ -36,26 +36,14 @@ const d_emb = 1152 - const vecSum = (xs, ys) => xs.map((x, i) => x + ys[i]) - const vecZero = d => new Array(d).fill(0) - const vecScale = (xs, s) => xs.map(x => x * s) - - const boxMuller = () => { - let x = Math.random() - let y = Math.random() - return Math.sqrt(-2.0 * Math.log(x)) * Math.cos(2.0 * Math.PI * y) - } - - const randn = (d, sigma) => Array.from({ length: d }, () => boxMuller() * sigma) - const K = 2 let candidates = [] const select = async candidate => { candidates = [] - const direction = randn(d_emb, 1 / d_emb) + const direction = util.randn(d_emb, 1 / d_emb) for (let i = -K; i <= K; i++) { - const newV = vecSum(vecScale(direction, i / K), candidate.vector) + const newV = util.vecSum(util.vecScale(direction, i / K), candidate.vector) candidates.push({ vector: newV, results: null, i: i + K }) } await Promise.all(candidates.map(async x => { @@ -67,7 +55,7 @@ console.log(candidates) } - select({ vector: randn(d_emb, 1 / d_emb) }) + select({ vector: util.randn(d_emb, 1 / d_emb) }) const handleKey = ev => { const num = parseInt(ev.key) @@ -75,4 +63,4 @@ select(candidates[num - 1]) } } - \ No newline at end of file + diff --git a/clipfront2/src/SearchResults.svelte b/clipfront2/src/SearchResults.svelte index 7a83a83..d0bdddf 100644 --- a/clipfront2/src/SearchResults.svelte +++ b/clipfront2/src/SearchResults.svelte @@ -28,6 +28,10 @@ {#key `${queryCounter}${result.file}`}
+ {#if result[5]} +
{result[0]}
+
{JSON.stringify(result[5])}
+ {/if}
{/key} {/each} @@ -72,7 +76,7 @@ export const redrawGrid = async () => { if (refreshLayout) { refreshLayout() - await recomputeScroll() + setTimeout(recomputeScroll, 0) } } @@ -82,7 +86,7 @@ } const handleScroll = () => { - if (window.scrollY + window.innerHeight >= heightThreshold && pendingImageLoads === 0) { + if (window.scrollY + window.innerHeight >= heightThreshold) { recomputeScroll() if (window.scrollY + window.innerHeight < heightThreshold) return; let init = displayedResults.length diff --git a/clipfront2/src/common.sass b/clipfront2/src/common.sass index e5ff03c..1c0b390 100644 --- a/clipfront2/src/common.sass +++ b/clipfront2/src/common.sass @@ -1,4 +1,11 @@ @use 'sass:color' +@font-face + font-family: 'Iosevka' + font-style: normal + font-weight: 400 + src: url(./iosevka.woff2) format('woff2') + unicode-range: U+0000-00C0, U+A2-A9, U+AC-AE, U+00D7, U+00F7, U+FEFF, U+FFFD + $palette-primary: #3f9b0b $palette-secondary: #033500 diff --git a/clipfront2/src/iosevka.woff2 b/clipfront2/src/iosevka.woff2 new file mode 100644 index 0000000000000000000000000000000000000000..a4b67c5d2260ffe27f27a205ab9d0d2bf48d7cef GIT binary patch literal 13844 zcmV+vHtWfEPew8T0RR9105%i=4gdfE0F=l805!b;9~Te+00000000000000000000 z0000QG#els%uohk0D%G#2nvD3Ac2%}0X7081Bze^g){&JAO(VY2Ot}Bc@^9ky4?;0 z)J2XligHE!4WcOBq^)NEe^}7Q2*LUtT5SQUB-SW~M)91hZhTJEASl{THs0^dROze^ zp=dS)$FSScYz(W23Q_Sp-ZPxKS^n!nfxSIv;5Z*O*CJlGpS^ntA*B(DF(d&o#(bpJ zP*lD@6z!k}jIjYKmfo>6EDf=BEWO~b^1B5~BRv288s)jKhoFaOQLP1_q&yJHxagE8 zQX2_K!cN6SnDF;Mb-Fx}Eg63t*7Y1Bm5|2{Yc153yVl*q&Z&Fo8lZ|mE7{hQKL}me zZsj(BP$fkb?vmm8x&5Dku*iFNcXJ!FX2WO&lo7j>yN-}Akp`s?lo1L>^8gXSfF}>Z zsE?wA0fO0}qg0H-D0m9y)tYVr=;d9=qT{oLsPI#MDre$h}-A-f?K1ni&AOv8G+dY0cCYfz?+bE$Q6Nzxt{?Ami-H#y- z0C{}EiWfjVHR@>oPx z)cN&zI*d)Pn~&Vyi0-!-sDZ|LvuGPo~{@&IipPF?$>8U4?Q z-P}#}aQiTEvioD4{$#H4}5=Inx zMYZ?_!mf*NyQO3>EVuXV*9RN_2MNoe;{McbCD`vTtMb8&J_B(IkTM}Zhe>#ol95wT zQc=^;@}c9)kG}wQ=+$Kq$jHPjNU(ZB)MwFvRYNvE2-QfKa1kO!i54SPoOlTmB}rzN zB2}7n0O63u#+qpA1RSCNk79bd=QlMZkdX?@QP;W%es6#YVZzXXmxCjRB;sX5uZKqt ze;c_lUhXKa3pZRkPihi|D~q97a0Pq}z5&0KPLzgAjnaJSZs{8YN9+)PWIl2Vc_%Z< z{*j$PGqE!K8vdMsi8UmLEF^DHZd4?7M~=!@$j>OoD|RcVEB{hfDhI4MR;blPD^IIP ztF2b|RMx7)stc-a)d$tj;qY+H@Z{my!;6O>8{R2S;t3-GvQBo$CHbM5)IdGdM;mD` zU8iS;)W!&S0f+=4sP__tdxgBPh>_B6^4+8RLx|C!1;?BW(+_ekjgv~z35WmefRH7I z?5oS9=}atkS6H4<=QmjxU^2#=hzR55;}lkjN+p^jGe%}4SY=XG#$~f2LWBq-j1Xaj z2$`gNuM?_C_U2sfsc_6S07E!J7*@Z`!RoS5F7W08KDDqqNs!TAf(;GT%H*q+A#1Hg z_?V$*!%>9mT{fqS&qQ6pHYJ~GD4L3zOs z_u3W@u3l~fy@+ggBw93ZV_%lseB4a8ZPI`cp*my?0%1Pp#(#t)ejG6mykk9v!0Jrz zK$+dAio0obcy|^Fk@anN287~NBq9POtYc}E6|8!YacF6kPN^P{s7F+Fb*bzq^cY5q>t;pqxaB2oYRzFR*a4%OQF=57k@y~DbWtgEcC${XB{Jx(F@8dz zEzEN}F?#RUTzAuL_q)TF3K=BBm*b6q?Dt==S-DEbIs&q`XOsdJ_q1rzOLYBmLbFyr zLl3jtUJ2yi{+tWADYlhD1eVVb2B`cZIR(|3zG*sl6~sYY&>@`5o8i*~r3p)#Jb=L$ z42&kwfzKgQqU)2A(MXY|6PM0|Y;Bxm2NNJhIur<2yI^`2HY4R$Nu|mhvsi?(07h}( z<_)XlmNv-`Z4`!(n|jHngjcNt8p^vSoKY~F1I46tK9nX7#bP^x!fPQyR$fTUYPHg% zC4kk-fh3|DY~yxViSj~uam@ns+W(!^U`f+{fl_>UjdX2aA12EZY7Q8gwPlz)( zL%9INPt2n&7eFmw#sF4nBr%Zu+QTAM%rcz_NVEmJD3I;WYJF3HXEdotTRU-$O|i`? z(<>u0Gi5oKzfxODW@L_XR&fo*=G(E-!I@R61gnP#jvjPpB3=eWA0FC|Z!!Xmt>H^+1GKnOl7EN|6@=vHS&OCCC$?R3t#LW6ZvcS@2W&HFY&y6TJa?aNtVi z>+ZELGr|9*1pxC>fn$qVY)^{=fmDq{3}gwwY+7tF=UEUEgKeH4bb1xNX&BEM%eAmZ z(u%G;u<(zq<7J8RWY)re|Is(p*w+HwffKyn0}JIXQIbS~ZKlR5m#T8EAWwJ*ED^fF zgoY%kQ52;{Q?!s@4R2qPEKa?QG}8*@4myWCK9{r1&W;JMmD%g|Pe7Qj8R76F;e>q6 z39D{lqumN)JG35Dv}s!iqMc$-jc!znFd3W%o0?u3o4F~=-c*;3rLq>~s_C_&%IhGG zW{bh*2kQoZVY$ddppB7b^fbH#_Bjq+41@r=Nx3KlXZ5-;=moGo75~R6eL%iOy^+BtHz~VkLp`f{h9aX~= z0)*OGh*NVqNX2oZ{Gf=_RZR7xZ7W@j!xF2z7Q}w?f{UGoE|htpQjB; zPcXk(?QZ$I`*3<*lKd0?8Jxs9^B1ptc*Pg0{Hz>omeD4^i7j7_CCFu-1GF$7`ElCS zDrY@xc?~PX?+-RxlJ31p^QRggQv6z(8!wIW(WP zwCUrG1M)J-1HU+P?0Ff6^~NS2m?J(i?%@EZN?C59GpQsWJPNQuge+I`Zc<2$rxfYP zW=b|kL77y}U2V=+F(!^ZFGCqd*)WbFxGeD$oFZ|QKuKf=i#v^HR^-l%yVL|qFp?Ys z#l8xNc{qaM=*VJ#m{#a{t9+o_qUMQ-kY8kxlLiH~X)RfN4>qyMcanomBnUSYqn|n? zrWV!(C;%t4uiq&PE?@iJZ2Xsj;k0_|91tvZF$tQO%Gob@F*wK42`BBb6P_{d8ZSel z?2H)Gtpm91PGcg<)^a7JxvSRtL8#8F4%3E0rqn740u~hG&U<7+j+;h)%c=)cfRGop zc-+Z?HOzDQI%WdE0q8n7S%yo$s0xZj!NN$m0U8T*6B6K;80S)14Kj4A(R~uMj^&KG zp?h_YyU?QpG!_9G90_jvRf;>uR{yFlx4Q@9C%F29O;9uHQ`|

qIb~l(i;;eo~Om zcEeh6cbz3}j@Rlc)?!I53DXu~^R-p-9#dVA=8^z$B@aOLd&omJWQWP(cq7UPEO76YyYNGtuDHI1~RuzRqi!z zx@L2dC;|6fM`T-8Nmbes$cO} zCQj)9F~OWBTy^zJ4Awg4A9h`3ng(<(8Op$5&b?jSGq{bmsypm657|=Sf%d@`~P20=d=xW8g#kiq+x4le~Nq*(GJzs*$NP5o<< zDV}`E7evH}VKCv!lmt~#k(taOLe?^cTdU*c8!1RhYT_gz9f%jxf~qg9oLJBjJGu2e z61r@R3lFP0<6*W>d9>^spoOEhtfcw^l^06V!-E}W% zxRYS)cfi31mVb@ON_J8~e2A!-tO_BW&$?6G#}iMJrm;F2Q?=@%8Bep^idJ?7QhZk= zvdyc+q(@UoN^P~kH4CZ9d?H{^*EPVrGR#v!Rw|(`gJ9!WEMA6- zjk<6|w;3K(K#XZ8QL1XygAe`(ZrdC6jto5yk z23ejoBEg2HKHsW%0VI&~iiJ@$ftpDC!Vr2ZKcCL*g=)TLaj`vv?TVVXyKlgtXXEKT zfF65ugzV6>6)x7Gz6qgierh14{Hx$MFLJ7%ykjwCr3Sg-tBh~cLIud=@vDg3wK#_$ zH&~EMAOsMt{|2=FOR&P{h+H8Zh8pGNO?P;^7Z#T}SvwvViITCmC*RiqSKhy;;%_!v zLqo$;^7e)u>9C=Q$0Y@cKsE^8w7O4+{yH|eikJc<7Z|(0r0DQ1bNm zCkWpKnL6`p6{jYX%bNZA<_1bx)<1ta_@ExnzlPIAzfY9?evs($hut)S(zZUu&nb7XW)jD2uZn=AS5c_+niEahwP|k7Z@je z*UbcUz>REgqOR+zF0x@Gg(a5~rkK-^jZRbv*p$tROJih-u@WATLA6^X@%iMNQ*a?E zQ;yp_lV%l12KY8vj@4VwM5Pwkt1DPIB3DI}aAAU5EIu$ote~XTWgr`gvDBj%VznG! zttiP0d7w@Ph6lKtkKxVz@jzgrN~}B?XqY_2QY6#UqncIaQL@uxOPe^X(tZ5#dG2+) zUUS}2S>K!@P+chTs&Vo(!*VP*U(`|b*GA5mz4vy5;!T@Z>+?lzyYsZJmlFDVCEWQ< zFK?s?o(L@qY7I@D3-zsTAbTj9l5{D`wDj8^@0;GOEMh|Hu?;i(_GT*zH%gQ_bBC&} zYkrl6ami*>t#^*p^+hyU!)V(`1j3 z6oH@oV9fa#uq9qxgug9KZW<$WH6OL7`8VgJrP`6Z%950m4r~V?KB0=GZ%U*AjvGPFo29pW2Sjfv`GyVMkajIsad8om}g#$WKfiD zi6!ZP2pTPEZKxSkfv(EX$c_grC>=@V{D{1$@uL$7Mlrid9pUD$7w*VUaw+eLV}0~u ze{SplM^s|l|K?+fj5>l>=80HsS_EDLyX%1HA+JL!S$yi2tm0if53WPG3G)N7)mYa9 zL~FYTrDQ_y?R-vMiz&)SN9*EYGkK>sCZl9Lgsi!BbF(B8*7#yg!6=ee^QZ=*JtGcgmM2Zi(YN_B3J>vW=~XVh5X2B5pIi&|yvvQ{*!6v8H$j4qMWGOVS6b zKVtdL^Vw()?E9mtP{T4oHMHM@Pc+UCy!q_b6S$ZNhAgk>x)~8 ziH(?didjx?6t+V_VLA@O4Mf{UY;}fmX=llDp2X0aLM^37Do+Gs%hR~5EK69~ONRwY z3oYpuY&t>hk0?}KPGs(FV|QjYFJY3VBCf1> z4NBJfRXv;?;0ixCafsBMO*d#iyc+$!fl@|f6dIH=YYbupl!hZi^w*K;ofCGtijX?e zx@Qq>oxd02s1F}|t+jE@Xz9_6mUtp!o&rncd zd83BD-szoq8ul8Sch4bZfA1>6E0wMF!a%ttB1t&W{ewa9e&uDZV{qm;z)u|>{M z{N&cN=)xj$jGLEB5?=e!rvoaW7L}t)lDp3%6pPL=vbNH=tJqyW(wLon0$3?ZMM{c; zLI=b9eh77bbFN#$W-kY#2AQ6}@!WBoF-xw2W@iS7j3iD*Z?q2eDxnkDn&Z5SHkNyD zkSRZ}dJ&Y-nyYFgk;q=uHO_d#$#cAUS}@jcUa;W#jc?nKgTiifmf&G`1x-k`V6c&; zed|wKr36}5*Eeo=9L6+>wb;?94~f-VZ;8=b^QqDYajYYYj*PTmAAM$N6xN_@hi=tT zOUx@R)u3ecO7C7L{jmBfQcx?BppPcYY+b_(3{Hvs8}}tmRNh&xHmRK%7Pq~XS&!Sz zzDt?+0A*cGg7Q=%9QA14%luIRX})S~S+AN$67osPlLATC6?$y<35NkYY-uNDEiS8T zidMsCx{hDIp&Vx#>WFB3CSF&YyGf0i-;8QLTe3PM;0`A~pwppqu~|lkXw_Jzxh(IxJxo%@U1&!(1O;N?KyX`GVLR*X zH*|;9kCSirzcY7i{N_IU&@}h)$@%j}R@faX+wrv*uF^GrC3!w&2+RmhNJl;(JSM_sBrC^13qVZSN|y`?Y@&X#g-uot@|dm z;yO)4-nk`ZE}S<)cC_ILF`)Jct?tV;3z5N9gmzl`F5IApSDi!&#ll!c7a3F2($P>o z16N9+dwAo&Nc#Dajo}J%>=0~EX!B|pok*vRo)UQVE@Hk;orc3FMm$-B2D_b?#KLrq zxn>RVDX}`41r!Oh4UONOSNk4ZILW8<_=(_8oeU?a%4mvIEobJKUAmS`O)G2}->=9U z6uHNH(tP3IqafR5l@96>DMbgC(UQ-oku>VD#5Q|U3Q2<^x+h{XDRw6g9Zo?)3K*7M z%pfJHQu0Y^`{CHx{CJw?8@D+yEud?9HJly^AarD6DJDd9gcN%|sQ@Maqa}e9o#jT) z#Mi!=u)DvlBG{rl8mRZj70waMMKt`-feTJjp~TU&zXrpji4E#W+svs_>?Pl^A|*wT zA1oBc2g98#1jg!H!@nJ5OcY|(5q+wBlEUwV0`BAf%&FSrXG>F+R4zWMAJQ7ANooL) z*JSQQIegWm)7ef&+FNHkRpE~9ez8zSQk29ohUfnUvEjMgf1V zRnYq12#lZO646e9o#Lz?CNy6rDzf(bEs0S6%TdyeZ*mt$iZo^wWpIDE^r$Gj;`l4C zi_v9Uc@vFdwGiVey;I((Lkc^Nb0?P_m1JjJRayK-zGB!-GVmIW>QO8aG zG9t9f%YVHf$*IybSMVB&l?V9dn5=A#gbvu~Kp!o6j27lGuA#Ck5|cws*eN40DENt} z#>P86$(D+14J1&P)~5dXED3r<&d!cB8U+phOXho zqUhxDUn8t`;H{~6A;ws|AZ3DA{wc(%=N>NWH(Tb0wbwWL(rtNlyp`?Yni)VbXo}0+ zG8PUEEz?U%(F-B0b_x1eZ7??s-kK;1V}B{ItYM0d6js!EUj60^gzVh6*sNM}Rp7=Q zI&`wJ&Af-{QS(K|9rDGZe23$rKD0z6Ua|~W1l5Mpp?6)VmZ@Fj+<52~4%}UI7qq1m z^k1)HR6Is3-Lqiih!5|LM~jAAxh%_XFJKCuRjiz3wf~P~wSGYQ=GX4F+L2a~hYma| z@J4T$-SMx2;Ijc20`#yr`3wxV=MJzzVFG$h{vADpewSZEmp3Yhj#nA2}w) z4njH9o)i4?RO&NIIliz|soYgaDy=5;pPKDB-!Wv9dQOn*b~U5+UHU(@$KZ=gv8uE? zg7d#`1z(Yg|EhCCBEE?)lSucPdFVHY4#ddjr(b|avECPr*osJ^{^hRf0Lqj8W`S&X zaA>f8dGV39jy~x~=X5>MzR1+jI7cJ0LaTK&(eZVsg5_0{u|y(X z5%_n<>%_Iy)|kgX&KCO^p^*-BN|#f0U0V6`XabX|rr%-4O$%|~%ncIB zhIyS5$x!HW$rva=L67wS^%^5>1Vd~|=e%Qlu?)yBGQiI+87|)l%D>OokU`C$G7P?b z{?z)dE7H9>p@M8r2h?E@gw+L{wuRC=|Lv~*{{w^1tJf#JIJS}L1zHv*TWem#pF96_ z$%E$7dOc@FOkn`N{#*t&_8;e-9Zz8XV!oM}RbFWHCXV zKb*eg;L9g(6x8lzp|Aab`rHS)&or~oRNlZeXld;^@Pkco8b>TS!e&HoJ|0F9{r}Cj zsHMp0HRjv(QmM}i$c$MLP7B&{lPE<|`zI_(p-39@=~*QhV#{+sOmfIlOp=5tW~zgH zLJo$`z52(NoORlwq-+*dKJg|MSq_sv3NZLr>EdxVr-12-(c`Koc_Wn`2=)jk8N~|WqdbJ6|OslOEYB1%vF9#Nt{_u`g z5v}Dh=Pj`sYxhP4E;Ti!AV^A+Ns22P-Qf(hNx(%ZEoUXetmJ6xqFg}}_{muCmzJ;2>>ExFWxT!_p1-c`uZ zLwCASOU9e1O}1@dKtt<3KBcw+?*7BZ4EOI&qUEL<-N89FXXw5#TGE^y_kt zF_K}VF~+&ek0jU^J(pAwmW+V)d}_gnXQnD5jHog_Be0fFJr@7BB+rX%pzWYgUCKGk|aJd|Em9l71;u(jc}sgti>KLEnl1 z9c<#~`~&X|ILpA(j)z&IX?3Y)OC;=SmsX8r{d~HqSZ7SCT;vY4y^OX~%k(l~onat= z&I`_?qkldcz;iY+xCc(zfm<>H+H__IDtT2({_8F=<)(UMY(DugO zieWM03P#P`vo_}&;MeIBvxW+z*2MU9`cdoO7*YpW9`(Pafx;loE&rpI(A}R)VCX$U zCI4^<%B19(K`uX%sd!CX28^HAAF+oa`y=}EI2=C`6x8FdHmkz^#TVU^^R@N^C+kOz#;g-~s6|mm0zMs*}Xo;&43!B6cf^ZDGi^ zSh1Uc7L5dhYy!Iwn17UDGa~GV_$G!uWc~CfwWF?fTv?75){%Hs%E)6Jg80^Gms|^Q zM6~| zV^2g6MhK%SJ{X8>!tmi>l+a0Mm0e#24U`ROYF&4+q4hfszOpI*?y&|Vl}#B9W4rw+ z0&DclsUj>Gj`e(M#rC$=PiVEhNw%#M>e&$1`l$yqXGBC*j%Oq0drn4z5CiNzk>jzX z%sbdH+1)WtM8EotK;|Ky2!p-d9bS7s(wgXP|H2WVwltMaO-6iQtgy^ONJ_ zEytIFopG!Gb(HwU{PRJJYZ)!1shm(c2tk9T6Dl>37Q}6OpsdvQ>aVtbj%j+=M1lZ% zWAIA)cKJ#gXgG|`zirZ7f%v3X*R-kKUI)bDYa!QjCyDc!eEz7?34Gp!(oy`iCLTWq zv=!sXr#Kv$^7SrKr@E?aS@OIaxSMoWAJgo5q#_F0*10Boxvo!d3#^fAJ|{}%`h7l% zNqz6R*EvWT@$^4y#E5poK;{@J0tVUW?h5)yA8NsHI0`txuI$-Pt97ouPfz|J%?2T-oS# zzqM~7F>!3O*bb)S2z^8>BxY?jU#t)&(cb@!6<9LfXwK`>$xWb#t#HYhWD;<*iXGG~ zJmh4E&}0P7@?g2%S=RQ~hp=aaH9L%BkjDHB;8<>Mf)PDW+7XMp0k)bp$Ty%&B^sDn zX{-$Fd<3+#GRKNG{`?&WC@P~uZ6A{T6jp})PExg`TD1nY7J?2Vb&{lMbxdi5vfvy0W z_Ja&OPjK4KA1}+x&sma^vUGk<(lv7n*81#Oj%p{L$&>y3{R5`!)jpW3arL6uWx1tJ z-?+v>xa9}DaIEa_{3SV%`y1`L84G}rnOAgnnB!$rUJU9NVey<=pARCVs?A4!*pT{KhS1zni)XW zAp^{6sW~H3Sud>YpXYrrFT{mTMzhgm+68E7yVm~El;=#3nJq{{|KSk{h;mI1p)@l$ z(~HClKDSE5Eu1J9o8xi#g{Gt>HZLF=l&v@zIfW#rL>>&haj}#nOE2CC+_~I)PO=59 z+hnqBtqWv}r9pdoskSs2=d3F$>uG4R)@@N-HXXFBY-Aes5*i2ww-SSmwkfqkoZnNT z*YC8VSk>JJDDGlied~0#IQtkb%eP!D_%TE=Rnmk_C)La=MXYyr=s!AHKH;}vBD z4kP^YZVTa`ddEngN|RWp*s97CQg&Pb|Mz+$mw(Shp+7j$iOlNfztvWDxUQo&F3#~Rv5gZZ8;R9?Vc`_U!g z4;K?f`8A7#>r~r>EIp@!SLx-7mH9&oyk(nB#MI~hPFHl`hnsvs-5j}@o<#d&wur=6 z9sU(d2Raj^G~~4PE-YMJQ!W%_sWXx9u3h?6N^Y1pnu1p;%fDf7G92Wu$f;0_4EtJyfY+W4j8}eE!gO;mLQ)WJ8eV37*Q7e} z$LSGi?GR0yyVbROB5!woedn4~VRCMcbdBmY%G{ZFt;*y;) zGpP~lA0Zdc{U$x}NVYfVxA#9X20pTwgv8gpyK@y{Qyy)S69&l(*^kN8YpB1RkUi|66Hev&ye(-d*~3K-iapzbH|z<*}&F73t!XV*I^fksr$f ziRX%g^lrDAQktMo7)Os=;VA#zMX&ba>E+vO)(N4uUmh^z@%}u-vm1Mxt2eRf%B<)8 z64~GC5>uEA4SQLlE(b$yvOLGC|H5R76HaTc!YmfceMJbfBjQBFm2VO_;G|@;gH#sw zUzKAXqJw$`1`TB3-+@HUmIWen?l8z8=mr0I25vfajV5U@JRW&vPy$as0hKyRWwX9kIpoP?c@9-yXUP;NomO9!vH&Y> z1$P)vif}3mNg)3Y1l{2Ws<}%IrAc*Fut-6^|c2%*ukUqc+&?$kTs??K&4l`@4`Dj_7qV;Ne@>=hos*?8rO(K#gC%dD-R zVsmw5u8BE!JsYw~3iZ+h6g#)NpqMVc>xP#t$1z3fl|Bf@aXrdbv_f&ItfLMmZU6^TExo-Ku(KC@ z;h?dG0ZhNbs|CIXKzC!Hlf@$7+0|eThku7UUZN*hDSvedMV4g*ij%Py4$4MxD%F*` ziJ!*Ezz`yz!hjjbbL=dFk(9(v_8mZwIj{LqUi5uY)JUP>(5R%Ln8lni)s@OQ4Y*DX z{*p)}AECU!GE~JXTPPt4Rw(PI^Oa;5?N{E*QCIRx875@nS^1d_no#=E0rY?PaS;P) zmBjh-#^yZPe{i5<6Upkr?k`m=T;?-T1~uR+)PvqC3jLu|6kvf|`(g1=zM1nT#UPb@ z2t&blw=_bN=)s;+&xetn&r4W3t_EC%hS9rChK+=Z469XtW6(WI7ocnzL8}`erViT- z*o7S9_Xqwg4w4Y>EFCq4b8CP_zvhcJ-rV%gu@?fX3nWZ~2_K>}p{#g$Oi9WgRxt~9 zq3$lzf!q}#UK9{BD7`Q$O5c-uR&?MIaWxWf6{*SDmHCyw2Nx*p57oOzA6|V?(W=j{ z-}Sy%iFOrul(Bx*iQ6NS?TI6PvC2FOc8OH)d*8kOyyLBYdG+DZJJrEKpq0S$2#D%) z1IxgHl55W-1V(U2K?IGSZEV)DGUd2nb_c~e@KIs%?`ume_g8dTP+|S++WWs_8NW=- z4_ol-tN-O(mJ|4cSzj1bX8(&DyI6z60O#jhk*nBSx&aDK^dzT3rsL;p(nJ*>_&R{ovnO zob)MGw_ho_e=z+(V0qml*W$|rMlEzy9yhQ2aFFx$-svxbB!7&te_CLd*fYT0du1Nm zPJrN>>+hI+BJD~yO-}x%2K~AQx*;CW@$m!kzjWFUEHn6GQhU4GERemJ>#PYZ@^e@( zCk9En;09rr2dZ(*PX?s1Hq3&}r9i);UJ!)yy&5*~F$R?B0A6F$o;L6!TMLn{v5&Yt zSG1f4f(?0GrdyW)xTLb39wsWK3rj0ONy)!p4onP&+{B%_La}&*cc-}bdv8b=>N>$r zd$Yx-K6=3VHE8ASCbSi8x?PDP1vg&G%%_;~g08E1{K;n>`K^4%vy=He_01rj*%Wix z%NqqCi!peI2hy8C{S>2v5tPRjYS({Zq`OS>IsDbbwZr28%!170Z%xg+@M-TSnD1TX z!{e-%c41ok3WcFH$)UFN`7eHcqx5osTiN+{sS0T>S&msG!YU7 zJT#nlGupYNDHr}@%Vif0=W!V)V4wC>6=Q-q1JD{5E8rUhIog1{O-9UbNMireu95sW6 zbU6BmTI~aW_G+NSxoD&c8X2~fWwzL2i!HX;Vv8*R0ASOm%@QbaE*hzVMusm-@MQ^< zI2VmnLAys>f$>atK!>|h)Z!6H zKn?%`0DhEDz{mgF+g{(&-5q-8PhikHm6_2A*xm-oQEIGe1|f_cdeeTKUXNwx7|G!n%u}bE;F4y->j+|Ec(mt0ZV+AWs}>&9O>dw&5X&VMm0>%K5FA)#8OJ3RjL0AqpK<^ zK__$AYWvmO8*ubgOLRV#E?k0lj&f$z4_Dji)YYJRTis`_L@ttUIk} zT1H#qnj)4`8n-I7uaI6iaR+%@wc_gcJZr{lcjuve43Jo60W9%eb2kCph0ih!+>ui53J3MjE?pf= z^A@?60c?RrsQ^%U)7-HDS<1GQ=MMy623wWXcisw%RFh0tk3K+oqgQ zSD5nhG95W^2A)NyTl;TCr7G0M8Zg_P%9C^k0Mk4MO-pxkOZG(#wSI2HLAdMZ+E zFtaETBE@R-b!AavgfeTRhrUJvJw?u=VAVzMEQB#@(wQZuGKXrmS>~ZRG{vC8>aLmI z=>{92e-!#D6qaU>ZAL?3GOa-b3P1qvNpkT3 literal 0 HcmV?d00001 diff --git a/clipfront2/src/util.js b/clipfront2/src/util.js index c7fb529..be0a185 100644 --- a/clipfront2/src/util.js +++ b/clipfront2/src/util.js @@ -78,3 +78,15 @@ fetch(config.backend_url).then(x => x.json().then(x => { serverConfig.set(x) window.serverConfig = x })) + +export const vecSum = (xs, ys) => xs.map((x, i) => x + ys[i]) +export const vecZero = d => new Array(d).fill(0) +export const vecScale = (xs, s) => xs.map(x => x * s) + +const boxMuller = () => { + let x = Math.random() + let y = Math.random() + return Math.sqrt(-2.0 * Math.log(x)) * Math.cos(2.0 * Math.PI * y) +} + +export const randn = (d, sigma) => Array.from({ length: d }, () => boxMuller() * sigma) diff --git a/clipfront2/static/index.html b/clipfront2/static/index.html index e0889fa..3da6bd5 100644 --- a/clipfront2/static/index.html +++ b/clipfront2/static/index.html @@ -5,7 +5,7 @@ - Meme Search Engine + Nooscope diff --git a/config2.json b/config2.json new file mode 100644 index 0000000..7c3a389 --- /dev/null +++ b/config2.json @@ -0,0 +1,10 @@ +{ + "listen_address": "127.0.0.1:5601", + "clip_server": "http://100.64.0.10:1708", + "descriptor_names": [ + "Useful", + "Meme", + "Aesthetic", + "Time" + ] +} diff --git a/diskann/chainq.py b/diskann/chainq.py new file mode 100644 index 0000000..0161953 --- /dev/null +++ b/diskann/chainq.py @@ -0,0 +1,164 @@ +import numpy as np +import msgpack +import math +import torch +from torch import autograd +import tqdm + +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +# https://github.com/una-dinosauria/local-search-quantization/blob/master/src/encodings/encode_chain.jl#L2 +# vectorized somewhat +def viterbi_encode( + out_codes: torch.Tensor, # N x M (ints) + vectors: torch.Tensor, # N x D + codebooks: torch.Tensor # M x H x D +): + N, D = vectors.shape + M, H, D2 = codebooks.shape + assert D == D2 + + # M x H x N - ||x-c||^2 ignoring x.T @ x component + unary_costs = -2 * (codebooks @ vectors.T) + (torch.linalg.norm(codebooks, dim=2) ** 2).unsqueeze(-1) + binary_costs = torch.zeros(M - 1, H, H, dtype=torch.float, device=DEVICE) + for i in range(M - 1): + binary_costs[i] = 2 * codebooks[i] @ codebooks[i + 1].T + print("binary costs", binary_costs) + + min_cost = torch.zeros(H, N, dtype=torch.float, device=DEVICE) + min_idx = torch.zeros(M, H, N, dtype=torch.int, device=DEVICE) + + cost = torch.zeros(H, N, dtype=torch.float, device=DEVICE) + + # forward pass - propagate optimal costs and indices forward + for step in tqdm.trange(M - 1): + if step > 0: + unary_costs[step] += min_cost + + ucost = unary_costs[step] + + # for all possible costs at this step + for j in range(H): + bcost = binary_costs[step, j].unsqueeze(-1) # independent of N + cost = ucost + bcost + + min_values, min_indices = torch.min(cost, dim=0) + min_cost[j] = min_values + min_idx[step, j] = min_indices + + unary_costs[-1] += min_cost + + # backward pass - propagate optimal indices backwards + out_codes[:, -1] = torch.argmin(unary_costs[-1], dim=0) + for i in range(M - 2, -1, -1): + out_codes[:, i] = min_idx[i][out_codes[:, i + 1], range(N)] + +def dims_for(dim, total_dims, m): + dims_per_code = total_dims // m + relevant_codebooks = [dim // dims_per_code] + if relevant_codebooks[-1] < m - 1: + relevant_codebooks.append(relevant_codebooks[-1] + 1) + return relevant_codebooks + +def update_codebooks(transformed_data, codes, h): + n, d = transformed_data.shape + n2, m = codes.shape + assert n == n2 + + new_codebook = torch.zeros(m, h, d, dtype=torch.float, device=DEVICE) + for dim in tqdm.trange(d): + relevant_codebooks = dims_for(dim, d, m) + assignment_matrix = torch.zeros(n, len(relevant_codebooks), h, dtype=torch.float, device=DEVICE) + indices = ( + torch.arange(n, dtype=torch.int, device=DEVICE).repeat(len(relevant_codebooks)), + torch.arange(len(relevant_codebooks), dtype=torch.int, device=DEVICE).repeat_interleave(n), + codes[:, relevant_codebooks].T.flatten() + ) + assignment_matrix[indices] = 1 + #print(assignment_matrix, assignment_matrix.shape, transformed_data[:, dim], transformed_data[:, dim].shape) + assignment_matrix = assignment_matrix.reshape(n, len(relevant_codebooks) * h) + #print(assignment_matrix, assignment_matrix.shape, transformed_data[:, dim], transformed_data[:, dim].shape) + #soln = torch.linalg.lstsq(assignment_matrix, transformed_data[:, dim])[0] + + reg = 1e-3 * torch.eye(len(relevant_codebooks) * h, device=DEVICE) + A = assignment_matrix.T @ assignment_matrix + reg + b = assignment_matrix.T @ transformed_data[:, dim] + + #print("matrix", A) + + usage = assignment_matrix.sum(dim=0) + unused = usage < 1 + + print(unused.sum().detach().item()) + soln = torch.linalg.solve(A, b) + + #print("solution", soln.reshape(len(relevant_codebooks), h)) + if unused.any(): + soln[unused] = torch.randn_like(soln[unused]) + + new_codebook[relevant_codebooks, :, dim] = soln.reshape(len(relevant_codebooks), h) + + if torch.isnan(new_codebook[relevant_codebooks, :, dim]).any(): + print("oh no", dim, new_codebook, relevant_codebooks, new_codebook[relevant_codebooks, :, dim]) + print("--- dim ---", dim) + print("- sum per column:", assignment_matrix.sum(dim=0)) # Check if any columns are all zero + print("- rank:", torch.linalg.matrix_rank(assignment_matrix)) + print("- condition number:", torch.linalg.cond(assignment_matrix)) + raise SystemExit + + return new_codebook + +BATCH = 8192 + +def train_chainq(vectors, m, h, transform, codebooks, n_iters): + for i in range(n_iters): + transformed_data = vectors @ transform.T + codes = torch.zeros(vectors.shape[0], m, dtype=torch.int, device=DEVICE) + for i in range(0, vectors.shape[0], BATCH): + viterbi_encode(codes[i:i+BATCH], transformed_data[i:i+BATCH], codebooks) + print("encoded") + #codebooks = update_codebooks(transformed_data, codes, h) + print("codebooks updated") + + quantized = torch.zeros_like(vectors, dtype=torch.float, device=DEVICE) + for j in range(m): + quantized[:] += codebooks[j, codes[:, j]] + print("quantized") + print((quantized - transformed_data).abs().mean(), transformed_data.abs().mean()) + + print("comparing") + res = transformed_data.T @ quantized + + print("running SVD...") + u, s, vt = torch.linalg.svd(res) + print("done.") + transform = u @ vt + print("regenerated transform") + + return codebooks, transform + +with open("opq.msgpack", "rb") as f: + data = msgpack.unpackb(f.read()) + +n_dims = 1152 +dataset = torch.tensor(np.random.permutation(np.fromfile("embeddings.bin", dtype=np.float16).reshape(-1, n_dims).astype(np.float32))[:BATCH*1], device=DEVICE) + +codebooks = torch.zeros(64, 256, n_dims, dtype=torch.float, device=DEVICE) + +centroids = torch.tensor(np.array(data["centroids"]).astype(np.float32).reshape(256, n_dims), device=DEVICE) +for dim in range(n_dims): + relevant_codebooks = dim // 64 + codebooks[relevant_codebooks, :, dim] = centroids[:, dim] + +print(centroids) +#print("codebooks", codebooks.tolist()) + +codebooks, transform = train_chainq(dataset, 64, 256, torch.tensor(np.array(data["transform"]).astype(np.float32).reshape(n_dims, n_dims), device=DEVICE), codebooks, 100) + +with open("chainq.msgpack", "wb") as f: + msgpack.pack({ + "codebooks": codebooks.cpu().numpy().flatten().tolist(), + "transform": transform.cpu().numpy().flatten().tolist(), + "n_dims": n_dims, + "n_dims_per_code": n_dims // 64 + }, f) diff --git a/diskann/src/vector.rs b/diskann/src/vector.rs index 0ed93e1..df98ace 100644 --- a/diskann/src/vector.rs +++ b/diskann/src/vector.rs @@ -45,7 +45,7 @@ impl Vector { } // Floats are vaguely annoying and not sortable (trivially), so we mostly represent dot products as integers -pub const SCALE: f32 = 1099511627776.0; +pub const SCALE: f32 = 4294967296.0; pub const SCALE_F64: f64 = SCALE as f64; pub fn dot<'a>(x: VectorRef<'a>, y: VectorRef<'a>) -> i64 { diff --git a/faiss_bench_quantizer.py b/faiss_bench_quantizer.py new file mode 100644 index 0000000..53a11ac --- /dev/null +++ b/faiss_bench_quantizer.py @@ -0,0 +1,131 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import sys +import faiss +import time +import numpy as np + +def eval_codec(q, xb): + t0 = time.time() + codes = q.compute_codes(xb) + t1 = time.time() + xb_decoded = q.decode(codes) + recons_err = ((xb - xb_decoded) ** 2).sum(axis=1).mean() + print(f"\tencode time: {t1 - t0:.3f} reconstruction error: {recons_err:.3f} ") + + +def eval_quantizer(q, xb, xt, variants=None): + if variants is None: + variants = [(None, None)] + t0 = time.time() + q.train(xt) + t1 = time.time() + train_t = t1 - t0 + print(f'\ttraining time: {train_t:.3f} s') + for name, val in variants: + if name is not None: + print(f"{name}={val}") + + if isinstance(q, faiss.ProductAdditiveQuantizer): + for i in range(q.nsplits): + subq = faiss.downcast_Quantizer(q.subquantizer(i)) + getattr(subq, name) + setattr(subq, name, val) + else: + getattr(q, name) # make sure field exists + setattr(q, name, val) + + eval_codec(q, xb) + + +todo = sys.argv[1:] + +ds = np.fromfile(todo[0], dtype=np.float16).reshape(-1, 1152).astype(np.float32) +print(ds) +del todo[0] + +if len(todo) > 0: + if todo[0].count("x") == 1: + M, nbits = [int(x) for x in todo[0].split("x")] + del todo[0] + elif todo[0].count("x") == 2: + nsplits, Msub, nbits = [int(x) for x in todo[0].split("x")] + M = nsplits * Msub + del todo[0] + +maxtrain = max(100 << nbits, 10**5) +print(f"eval on {M}x{nbits} maxtrain={maxtrain}") + +xb, xt = ds, ds + +nb, d = xb.shape +nt, d = xt.shape + + +# fastest to slowest + +if 'lsq-gpu' in todo: + lsq = faiss.LocalSearchQuantizer(d, M, nbits) + ngpus = faiss.get_num_gpus() + lsq.icm_encoder_factory = faiss.GpuIcmEncoderFactory(ngpus) + lsq.verbose = True + eval_quantizer(lsq, xb, xt, 'lsq-gpu') + +if 'pq' in todo: + pq = faiss.ProductQuantizer(d, M, nbits) + print("===== PQ") + eval_quantizer(pq, xb, xt) + +if 'opq' in todo: + d2 = ((d + M - 1) // M) * M + print("OPQ d2=", d2) + opq = faiss.OPQMatrix(d, M, d2) + opq.train(xt) + xb2 = opq.apply(xb) + xt2 = opq.apply(xt) + pq = faiss.ProductQuantizer(d2, M, nbits) + print("===== PQ") + eval_quantizer(pq, xb2, xt2) + +if 'prq' in todo: + print(f"===== PRQ{nsplits}x{Msub}x{nbits}") + prq = faiss.ProductResidualQuantizer(d, nsplits, Msub, nbits) + variants = [("max_beam_size", i) for i in (1, 2, 4, 8, 16, 32)] + eval_quantizer(prq, xb, xt, variants=variants) + +if 'plsq' in todo: + print(f"===== PLSQ{nsplits}x{Msub}x{nbits}") + plsq = faiss.ProductLocalSearchQuantizer(d, nsplits, Msub, nbits) + variants = [("encode_ils_iters", i) for i in (2, 3, 4, 8, 16)] + eval_quantizer(plsq, xb, xt, variants=variants) + +if 'rq' in todo: + print("===== RQ") + rq = faiss.ResidualQuantizer(d, M, nbits, ) + rq.max_beam_size + rq.max_beam_size = 30 # for compatibility with older runs + # rq.train_type = faiss.ResidualQuantizer.Train_default + # rq.verbose = True + variants = [("max_beam_size", i) for i in (1, 2, 4, 8, 16, 32)] + eval_quantizer(rq, xb, xt, variants=variants) + +if 'rq_lut' in todo: + print("===== RQ") + rq = faiss.ResidualQuantizer(d, M, nbits, ) + rq.max_beam_size + rq.max_beam_size = 30 # for compatibility with older runs + rq.use_beam_LUT + rq.use_beam_LUT = 1 + # rq.train_type = faiss.ResidualQuantizer.Train_default + # rq.verbose = True + variants = [("max_beam_size", i) for i in (1, 2, 4, 8, 16, 32, 64)] + eval_quantizer(rq, xb, xt, variants=variants) + +if 'lsq' in todo: + print("===== LSQ") + lsq = faiss.LocalSearchQuantizer(d, M, nbits) + variants = [("encode_ils_iters", i) for i in (2, 3, 4, 8, 16)] + eval_quantizer(lsq, xb, xt, variants=variants) diff --git a/frontend_config.json b/frontend_config.json index 7f27212..992a413 100644 --- a/frontend_config.json +++ b/frontend_config.json @@ -1,8 +1,12 @@ { - "backend_url": "http://localhost:5601/", + "backend_url": "/backend", "image_path": "", "thumb_path": null, "description": "Organizing the world's memes.", "name": "Nooscope", - "telemetry_endpoint": "/telemetry" + "telemetry_endpoint": "/telemetry", + "friendly_mode_default_terms": [ + { "predefined_embedding": "Meme", "weight": 1, "sign": "+" }, + { "predefined_embedding": "Aesthetic", "weight": 0.2, "sign": "+" } + ] } diff --git a/genseahash.py b/genseahash.py new file mode 100644 index 0000000..5f21fdc --- /dev/null +++ b/genseahash.py @@ -0,0 +1,4 @@ +import seahash, sys + +with open(sys.argv[1], "rb") as f: + print(seahash.hash(f.read())) diff --git a/meme-rater/active_learning.py b/meme-rater/active_learning.py index dac462a..2d8635f 100644 --- a/meme-rater/active_learning.py +++ b/meme-rater/active_learning.py @@ -31,6 +31,7 @@ model.load_state_dict(torch.load(modelc)) params = sum(p.numel() for p in model.parameters()) print(f"{params/1e6:.1f}M parameters") print(model) +model.eval() files = shared.fetch_all_files() variance = {} @@ -56,4 +57,4 @@ with torch.inference_mode(): top = sorted(variance.items(), key=lambda x: -x[1]) with open("top.json", "w") as f: - json.dump(top[:100], f) + json.dump(top[:50], f) diff --git a/meme-rater/active_learning_find_top.py b/meme-rater/active_learning_find_top.py index b12b435..d1411c5 100644 --- a/meme-rater/active_learning_find_top.py +++ b/meme-rater/active_learning_find_top.py @@ -52,7 +52,7 @@ channel = int(sys.argv[2]) percentile = float(sys.argv[3]) output_pairs = int(sys.argv[4]) mean_scores = numpy.mean(numpy.stack([score for filename, score in results.items()])) -top = sorted(((filename, score) for filename, score in results.items() if (score > mean_scores).all()), key=lambda x: x[1][channel], reverse=True) +top = sorted(((filename, score) for filename, score in results.items()), key=lambda x: x[1][channel], reverse=True) select_from = top[:int(len(top) * percentile)] out = [] diff --git a/meme-rater/active_learning_gradients.py b/meme-rater/active_learning_gradients.py index 13909df..a349e5f 100644 --- a/meme-rater/active_learning_gradients.py +++ b/meme-rater/active_learning_gradients.py @@ -13,14 +13,14 @@ import sys from model import Config, BradleyTerry import shared -batch_size = 128 +batch_size = 32 num_pairs = batch_size * 1024 device = "cuda" config = Config( d_emb=1152, n_hidden=1, - n_ensemble=1, + n_ensemble=16, device=device, dtype=torch.float32, output_channels=3, @@ -32,6 +32,7 @@ model.load_state_dict(torch.load(modelc)) params = sum(p.numel() for p in model.parameters()) print(f"{params/1e6:.1f}M parameters") print(model) +model.eval() files = shared.fetch_all_files() importance = {} @@ -41,8 +42,8 @@ buffers = {k: v.detach() for k, v in model.named_buffers()} # https://pytorch.org/tutorials/intermediate/per_sample_grads.html def compute_loss(params, buffers, sample, target): - batch = sample.unsqueeze(0) - targets = target.unsqueeze(0) + batch = sample.unsqueeze(1) + targets = target.unsqueeze(1).expand((config.n_ensemble, 1, config.output_channels)) predictions = functional_call(model, (params, buffers), (batch,)) loss = F.binary_cross_entropy(predictions, targets) @@ -75,4 +76,4 @@ for bstart in tqdm(range(0, len(pairs), batch_size)): top = sorted(importance.items(), key=lambda x: -x[1]) with open("top.json", "w") as f: - json.dump(top[:256], f) + json.dump(top[:50], f) diff --git a/meme-rater/ensemble_to_wide_model.py b/meme-rater/ensemble_to_wide_model.py index b357c19..b492857 100644 --- a/meme-rater/ensemble_to_wide_model.py +++ b/meme-rater/ensemble_to_wide_model.py @@ -31,6 +31,11 @@ params = sum(p.numel() for p in model.parameters()) print(f"{params/1e6:.1f}M parameters") print(model) +torch.random.manual_seed(1) + +for x in model.ensemble.models: + x.output.bias.data.fill_(0) + out_layers = [] out_bias = [] @@ -50,7 +55,7 @@ with torch.inference_mode(): downprojection[:, i*config.d_emb:(i+1)*config.d_emb] = model.ensemble.models[i].output.weight.data.clone() for i in range(10): - input = torch.randn(3, config.d_emb) + input = torch.randn(4, config.d_emb) ground_truth_result = model.ensemble(input.unsqueeze(0).expand((config.n_ensemble, *input.shape))).mean(dim=0).T r_result = input for (layer, bias) in zip(out_layers, out_bias): @@ -58,13 +63,14 @@ with torch.inference_mode(): print(r_result.shape, bias.shape) r_result = F.silu(r_result) r_result = torch.matmul(downprojection, r_result) / config.n_ensemble - bias_diff = torch.mean(r_result - ground_truth_result) - model_issue_diff = torch.max(torch.abs(r_result - bias_diff - ground_truth_result)) - print(model_issue_diff) + error = torch.mean(r_result - ground_truth_result) + print(error) + assert error.detach().cpu().numpy() < 1e-4 print("test vector:") - print(input.flatten().tolist()) + #print(input.flatten().tolist()) print("ground truth result:") + print(ground_truth_result.shape) print(ground_truth_result.T.flatten().tolist()) save_file({ diff --git a/meme-rater/model.py b/meme-rater/model.py index 49cf551..b548949 100644 --- a/meme-rater/model.py +++ b/meme-rater/model.py @@ -35,7 +35,7 @@ class Ensemble(nn.Module): # model batch def forward(self, embs): - return torch.stack([ x(embs[i]) for i, x in enumerate(self.models) ]) # model batch output_dim=1 + return torch.stack([ x(embs[i]) for i, x in enumerate(self.models) ]) # model batch output_channels class BradleyTerry(nn.Module): def __init__(self, config): diff --git a/meme-rater/rater_server.py b/meme-rater/rater_server.py index 9ee2080..36a95a6 100644 --- a/meme-rater/rater_server.py +++ b/meme-rater/rater_server.py @@ -51,19 +51,25 @@ async def index(request):

+ + + + + +
@@ -82,35 +88,29 @@ async def index(request): document.querySelector("form").submit() }} }} + const keys = {{ + "q": "rq1p", + "w": "rq1", + "e": "rqe", + "r": "rq2", + "t": "rq2p", + "a": "rm1p", + "s": "rm1", + "d": "rme", + "f": "rm2", + "g": "rm2p", + "z": "ra1p", + "x": "ra1", + "c": "rae", + "v": "ra2", + "b": "ra2p", + }} document.addEventListener("keypress", function(event) {{ - if (event.key === "q") {{ - document.querySelector("input[name='rating-useful'][value='1']").checked = true - commitIfReady() - }} else if (event.key === "w") {{ - document.querySelector("input[name='rating-useful'][value='eq']").checked = true - commitIfReady() - }} else if (event.key === "e") {{ - document.querySelector("input[name='rating-useful'][value='2']").checked = true - commitIfReady() - }} else if (event.key === "a") {{ - document.querySelector("input[name='rating-meme'][value='1']").checked = true - commitIfReady() - }} else if (event.key === "s") {{ - document.querySelector("input[name='rating-meme'][value='eq']").checked = true - commitIfReady() - }} else if (event.key === "d") {{ - document.querySelector("input[name='rating-meme'][value='2']").checked = true - commitIfReady() - }} else if (event.key === "z") {{ - document.querySelector("input[name='rating-aesthetic'][value='1']").checked = true - commitIfReady() - }} else if (event.key === "x") {{ - document.querySelector("input[name='rating-aesthetic'][value='eq']").checked = true - commitIfReady() - }} else if (event.key === "c") {{ - document.querySelector("input[name='rating-aesthetic'][value='2']").checked = true - commitIfReady() + const key = keys[event.key] + if (key) {{ + document.getElementById(key).checked = true }} + commitIfReady() }}); diff --git a/meme-rater/shared.py b/meme-rater/shared.py index 5c52c97..46caf96 100644 --- a/meme-rater/shared.py +++ b/meme-rater/shared.py @@ -20,27 +20,35 @@ def fetch_embedding(filename): csr.close() return x.copy() # PyTorch complains otherwise due to bad -def map_rating(rating, uncertainty=0.05): +def map_rating(rating): def map_one(rating): match rating: case "1": # meme 1 is better - return 1 - uncertainty + return 0.9 case "2": - return uncertainty + return 0.1 + case "2+" | "2p": + return 0.3 + case "1+" | "1p": + return 0.7 case "eq": return 0.5 case _: raise ValueError("invalid rating, please fix") return np.array([map_one(r) for r in rating.split(",")]) -def fetch_ratings(): +def fetch_ratings(sets): trains = defaultdict(list) validations = defaultdict(list) csr = db.execute("SELECT meme1, meme2, rating, iteration FROM ratings") + its = set() for meme1, meme2, rating, iteration in csr.fetchall(): + if iteration not in its: + print(iteration) + its.add(iteration) (validations if is_val_set(meme1, meme2) else trains)[int((iteration and iteration.split("-")[0]) or "0")].append((fetch_embedding(meme1), fetch_embedding(meme2), map_rating(rating))) csr.close() - return list(x[1] for x in sorted(trains.items())), list(x[1] for x in sorted(validations.items())) + return list(x[1] for x in sorted(trains.items()) if str(x[0]) in sets), list(x[1] for x in sorted(validations.items()) if str(x[0]) in sets) def generate_random_permutations(x, n): out = [] diff --git a/meme-rater/train.py b/meme-rater/train.py index 469701b..820c771 100644 --- a/meme-rater/train.py +++ b/meme-rater/train.py @@ -9,11 +9,12 @@ import time from tqdm import tqdm import math from dataclasses import dataclass, asdict +import sys from model import Config as ModelConfig, BradleyTerry import shared -trains, validations = shared.fetch_ratings() +trains, validations = shared.fetch_ratings(sys.argv[1:]) for train, validation in zip(trains, validations): print(len(train), len(validation)) @@ -36,11 +37,11 @@ config = TrainConfig( n_ensemble=16, device=device, dtype=torch.float32, - dropout=0.1, + dropout=0.0, output_channels=3 ), lr=3e-4, - weight_decay=0.2, + weight_decay=0.0, batch_size=1, epochs=5, compile=False, diff --git a/sae/shared.py b/sae/shared.py index f01fdbd..b1a73bb 100644 --- a/sae/shared.py +++ b/sae/shared.py @@ -1,12 +1,12 @@ import torch -import pyarrow as pa +import numpy as np torch.set_float32_matmul_precision("high") -with pa.memory_map("../../sample_1m.arrow", "r") as source: - loaded_arrays = pa.ipc.open_file(source).read_all() +loaded_arrays = np.memmap("embeddings.bin", dtype=np.float16).reshape(-1, 1152) +loaded_arrays_permutation = np.random.permutation(len(loaded_arrays)) train_split = 0.8 def ckpt_path(steps): - return f"ckpt/{steps}.pt", f"ckpt/{steps}.optim.pt" \ No newline at end of file + return f"ckpt/{steps}.pt", f"ckpt/{steps}.optim.pt" diff --git a/sae/train.py b/sae/train.py index 2c5e37e..46816f5 100644 --- a/sae/train.py +++ b/sae/train.py @@ -6,9 +6,10 @@ import json import time from tqdm import tqdm from dataclasses import dataclass, asdict +import math from model import SAEConfig, SAE -from shared import train_split, loaded_arrays, ckpt_path +from shared import train_split, loaded_arrays, ckpt_path, loaded_arrays_permutation device = "cuda" @@ -24,7 +25,7 @@ class TrainConfig: config = TrainConfig( model=SAEConfig( d_emb=1152, - d_hidden=65536, + d_hidden=262144, top_k=128, device=device, dtype=torch.float32, @@ -33,7 +34,7 @@ config = TrainConfig( lr=3e-4, weight_decay=0.0, batch_size=64, - epochs=5, + epochs=1, compile=True, ) @@ -81,7 +82,7 @@ with open(logfile, "w") as log: batch = [] t = tqdm(range(0, int(len(loaded_arrays) * train_split), config.batch_size)) for batch_start in t: - batch = numpy.stack([ numpy.frombuffer(embedding.as_py(), dtype=numpy.float16) for embedding in loaded_arrays["embedding"][batch_start:batch_start + config.batch_size] ]) + batch = numpy.stack([ embedding for embedding in loaded_arrays[loaded_arrays_permutation[batch_start:batch_start + config.batch_size]] ]) if len(batch) == config.batch_size: batch = torch.Tensor(batch).to(device) @@ -98,4 +99,4 @@ with open(logfile, "w") as log: print(ctr) numpy.save(f"ckpt/{steps}.counters.npy", ctr) -print(logfile) \ No newline at end of file +print(logfile) diff --git a/slow_dump_parse_script.py b/slow_dump_parse_script.py new file mode 100644 index 0000000..46661ed --- /dev/null +++ b/slow_dump_parse_script.py @@ -0,0 +1,35 @@ +import umsgpack +import zstandard +import pyarrow as pa + +data = [] + +with open("sample.zst", "rb") as f: + decomp = zstandard.ZstdDecompressor() + reader = decomp.stream_reader(f) + count = 0 + while True: + try: + url, id, title, subreddit, author, timestamp, embedding = umsgpack.unpack(reader) + embedding = bytes(embedding) + data.append({"url": url, "id": id, "title": title, "subreddit": subreddit, "author": author, "timestamp": timestamp, "embedding": embedding}) + count += 1 + except umsgpack.InsufficientDataException: + break + print(count) + +schema = pa.schema([ + ("url", pa.string()), + ("id", pa.string()), + ("title", pa.string()), + ("subreddit", pa.string()), + ("author", pa.string()), + ("timestamp", pa.int64()), + ("embedding", pa.binary()) +]) + +table = pa.Table.from_pylist(data, schema=schema) + +with pa.OSFile("output.parquet", "wb") as sink: + with pa.RecordBatchFileWriter(sink, schema) as writer: + writer.write_table(table) \ No newline at end of file diff --git a/src/common.rs b/src/common.rs index 3d089d0..ba77a62 100644 --- a/src/common.rs +++ b/src/common.rs @@ -183,8 +183,8 @@ pub struct FrontendInit { pub type EmbeddingVector = Vec; #[derive(Debug, Serialize)] -pub struct QueryResult { - pub matches: Vec<(f32, String, String, u64, Option<(u32, u32)>)>, +pub struct QueryResult { + pub matches: Vec<(f32, String, String, u64, Option<(u32, u32)>, Option)>, pub formats: Vec, pub extensions: HashMap, } @@ -203,7 +203,9 @@ pub struct QueryRequest { pub terms: Vec, pub k: Option, #[serde(default)] - pub include_video: bool + pub include_video: bool, + #[serde(default)] + pub debug_enabled: bool } lazy_static::lazy_static! { @@ -238,8 +240,9 @@ pub async fn get_total_embedding>>>, B: Fu } } if let Some(name) = &term.predefined_embedding { - let embedding = predefined_embeddings.get(name).context("name invalid")?; - total_embedding = total_embedding + embedding * term.weight.unwrap_or(1.0); + if let Some(embedding) = predefined_embeddings.get(name) { + total_embedding = total_embedding + embedding * term.weight.unwrap_or(1.0); + } } } diff --git a/src/dump_processor.rs b/src/dump_processor.rs index 38d9b2d..f3885f8 100644 --- a/src/dump_processor.rs +++ b/src/dump_processor.rs @@ -69,6 +69,10 @@ struct CLIArguments { gpu: Option, #[argh(option, description="descriptor CDFs")] cdfs: Option, + #[argh(option, description="postfilter by embedding (late discard if dot product above threshold)")] + postfilter: Vec, + #[argh(option, description="postfilter by scorer")] + postfilter_scorer: Vec, } #[derive(Clone, Deserialize, Serialize, Debug)] @@ -162,10 +166,22 @@ fn main() -> Result<()> { } else { (snd, None) }; + let mut post_threshold = None; + for x in &args.postfilter { + let (tname, snd) = x.split_once(':').context("invalid postfilter argument")?; + if tname == name { + post_threshold = Some(snd.parse::().context("parse postfilter threshold")?); + } + } let blob = fs::read(path).context("read embedding")?; - embeddings.push((name.to_string(), common::decode_fp16_buffer(&blob), Histogram::new(-1.0, 1.0, 512), threshold)); + embeddings.push((name.to_string(), common::decode_fp16_buffer(&blob), Histogram::new(-1.0, 1.0, 512), threshold, post_threshold)); } + let postfilter_scorer = args.postfilter_scorer.iter().map(|x| { + let (id, snd) = x.split_once(':').context("invalid postfilter scorer argument")?; + Ok((id.parse::().context("invalid postfilter scorer id")?, snd.parse::().context("parse postfilter scorer threshold")?)) + }).collect::>>()?; + let pq_codec = if let Some(pq_codec) = args.pq_codec { let data = fs::read(pq_codec).context("read pq codec")?; let pq_codec: ProductQuantizer = rmp_serde::from_read(&data[..]).context("decode pq codec")?; @@ -255,7 +271,6 @@ fn main() -> Result<()> { files.sort_by_key(|(id, _)| *id); shard_id_mappings.sort_by_key(|(id, _)| *id); - let read_out_vertices = move |id: u32| -> Result<(Vec, Vec)> { let mut out_vertices: Vec = vec![]; let mut shards: Vec = vec![]; @@ -323,6 +338,8 @@ fn main() -> Result<()> { let th = std::thread::spawn(move || reader_thread(&args.paths, tx)); + let mut postfilter_count = 0; + let mut rng2 = rng.fork(); let initial_filter = |x: ProcessedEntry| { i += 1; @@ -337,7 +354,9 @@ fn main() -> Result<()> { latest_timestamp = latest_timestamp.max(timestamp); earliest_timestamp = earliest_timestamp.min(timestamp); - for (_name, vec, histogram, threshold) in &mut embeddings { + let mut postfilter = false; + + for (_name, vec, histogram, threshold, postfilter_threshold) in &mut embeddings { let dot = SpatialSimilarity::dot(&embedding, vec).unwrap() as f32; histogram.add(dot); if let Some(threshold) = threshold { @@ -345,6 +364,12 @@ fn main() -> Result<()> { return None; } } + if let Some(threshold) = postfilter_threshold { + if dot >= *threshold { + postfilter = true; + postfilter_count += 1; // somewhat wrong because could be duplicated + } + } } // distance thresholding is too costly to do over a long range so just do it badly @@ -393,7 +418,7 @@ fn main() -> Result<()> { println!("{}", data); } - Some((x, embedding)) + Some((x, embedding, postfilter)) }; let mut dead_count = 0; @@ -404,14 +429,14 @@ fn main() -> Result<()> { let batch: Vec<_> = batch.collect(); let batch_len = batch.len(); - for (x, _embedding) in batch.iter() { + for (x, _embedding, _postfilter) in batch.iter() { if let Some(ref mut file) = output_file { file.write_all(&x.embedding)?; } } if let Some(shards) = &mut shards_out { - for (i, (x, embedding)) in batch.iter().enumerate() { + for (i, (x, embedding, _postfilter)) in batch.iter().enumerate() { // closest matches first shards.sort_by_cached_key(|&(ref centroid, _, shard_count, _shard_index)| { let mut dot = SpatialSimilarity::dot(¢roid, &embedding).unwrap(); @@ -439,7 +464,7 @@ fn main() -> Result<()> { let quantizer = pq_codec.as_ref().context("PQ codec needed to output index")?; let mut batch_embeddings = Vec::with_capacity(batch.len() * D_EMB as usize); - for (_x, embedding) in batch.iter() { + for (_x, embedding, _postfilter) in batch.iter() { batch_embeddings.extend_from_slice(&embedding); } let codes = quantizer.quantize_batch(&batch_embeddings); @@ -448,7 +473,7 @@ fn main() -> Result<()> { let cdfs = cdfs.as_ref().context("score model CDFs needed to output index")?; let scores = score_model.score_batch(&batch_embeddings)?; - for (i, (x, _embedding)) in batch.into_iter().enumerate() { + for (i, (x, _embedding, mut postfilter)) in batch.into_iter().enumerate() { let (vertices, shards) = read_out_vertices(count + i as u32)?; // TODO: could parallelize this given the batching let mut entry_scores = scores[(i * score_model.output_channels)..((i + 1) * score_model.output_channels)].to_vec(); @@ -465,6 +490,13 @@ fn main() -> Result<()> { index_output_file.2.write_all(&[cdf_bucket])?; } + for (index, score) in postfilter_scorer.iter() { + if entry_scores[*index] < *score { + postfilter = true; + break; + } + } + let mut entry = PackedIndexEntry { id: count + i as u32, vertices, @@ -476,9 +508,10 @@ fn main() -> Result<()> { shards }; let mut bytes = bitcode::encode(&entry); - if bytes.len() > (RECORD_PAD_SIZE - 2) { + // as an ugly hack for removing entries already in the index shards, kill the URL and make it a graph node only + if bytes.len() > (RECORD_PAD_SIZE - 2) || postfilter { // we do need the records to fit in a fixed size and can't really drop things, so discard URL so it can exist as a graph node only - entry.url = String::new(); + entry.url = String::new(); // URL is only input-controlled, arbitrary-length field bytes = bitcode::encode(&entry); dead_count += 1; } @@ -494,11 +527,11 @@ fn main() -> Result<()> { } if args.print_aggregates { - println!("earliest={} latest={} count={} read={} deduped={}", earliest_timestamp, latest_timestamp, count, i, deduped_count); + println!("earliest={} latest={} count={} read={} deduped={} postfiltered={}", earliest_timestamp, latest_timestamp, count, i, deduped_count, postfilter_count); } if let Some(histogram_path) = args.histograms { let mut file = fs::File::create(histogram_path)?; - for (name, _, histogram, _) in &embeddings { + for (name, _, histogram, _, _) in &embeddings { let width = 800.0; let padding = 40.0; let bars_height = 300 as f64; diff --git a/src/get_embedding.py b/src/get_embedding.py index 54bf9b3..37f1da4 100644 --- a/src/get_embedding.py +++ b/src/get_embedding.py @@ -10,14 +10,16 @@ with open("mse_config.json") as f: def get_embedding(req): return msgpack.unpackb(requests.post(config["clip_server"], data=msgpack.packb(req)).content) -output, input, *xs = sys.argv[1:] +mode, output, input = sys.argv[1:] with open(output, "wb") as f: - with open(input, "rb") as g: - input_data = g.read() - if not xs: + if mode == "image": + with open(input, "rb") as g: + input_data = g.read() result = get_embedding({"images": [input_data]})[0] + elif mode == "text": + result = get_embedding({"text": input})[0] else: - result = get_embedding({"text": xs})[0] + raise Exception("unknown mode") f.write(result) print(base64.urlsafe_b64encode(result).decode("ascii")) diff --git a/src/main.rs b/src/main.rs index 9767d92..46873ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -892,7 +892,7 @@ async fn build_index(config: Arc) -> Result { } #[instrument(skip(index))] -async fn query_index(index: &IIndex, query: EmbeddingVector, k: usize, video: bool) -> Result { +async fn query_index(index: &IIndex, query: EmbeddingVector, k: usize, video: bool) -> Result> { let result = index.vectors.search(&query, k as usize)?; let mut seen_videos = HashSet::new(); @@ -916,7 +916,8 @@ async fn query_index(index: &IIndex, query: EmbeddingVector, k: usize, video: bo index.filenames[id].container_filename(), generate_filename_hash(&index.filenames[id as usize]).clone(), index.format_codes[id], - index.metadata[id].as_ref().map(|x| (x.width, x.height)) + index.metadata[id].as_ref().map(|x| (x.width, x.height)), + Option::<()>::None )) }) .collect(); diff --git a/src/query_disk_index.rs b/src/query_disk_index.rs index ee40050..8fa5b84 100644 --- a/src/query_disk_index.rs +++ b/src/query_disk_index.rs @@ -7,7 +7,7 @@ use argh::FromArgs; use itertools::Itertools; use foldhash::{HashSet, HashSetExt}; use half::f16; -use diskann::{NeighbourBuffer, vector::{fast_dot_noprefetch, QueryLUT, scale_dot_result, scale_dot_result_f64}}; +use diskann::{NeighbourBuffer, vector::{fast_dot_noprefetch, QueryLUT, scale_dot_result, scale_dot_result_f64, SCALE_F64}}; use simsimd::SpatialSimilarity; use memmap2::{Mmap, MmapOptions}; use std::rc::Rc; @@ -18,13 +18,14 @@ use http_body_util::{BodyExt, Empty, Full}; use prometheus::{register_int_counter, register_int_counter_vec, register_int_gauge, Encoder, IntCounter, IntGauge, IntCounterVec}; use std::pin::Pin; use std::future::Future; -use serde::Serialize; +use serde::{Serialize, Deserialize}; use std::str::FromStr; use std::collections::HashMap; +use std::io::Write; mod common; -use common::{resize_for_embed_sync, FrontendInit, IndexHeader, InferenceServerConfig, PackedIndexEntry, QueryRequest, QueryResult}; +use common::{resize_for_embed_sync, QueryTerm, FrontendInit, IndexHeader, InferenceServerConfig, PackedIndexEntry, QueryRequest, QueryResult}; #[derive(FromArgs, Clone)] #[argh(description="Query disk index")] @@ -43,10 +44,18 @@ struct CLIArguments { search_list_size: Option, #[argh(switch, description="always use full-precision vectors (slow)")] disable_pq: bool, - #[argh(option, short='l', description="listen address")] - listen_address: Option, - #[argh(option, short='c', description="clip server")] - clip_server: Option, + #[argh(option, short='c', description="server config file")] + config_path: Option +} + +#[derive(Deserialize, Clone)] +struct ServerConfig { + listen_address: String, + clip_server: String, + descriptor_names: Vec, + telemetry_file: String, + search_list: usize, + beam_width: usize } lazy_static! { @@ -82,17 +91,30 @@ fn next_several_unvisited(s: &mut NeighbourBuffer, n: usize) -> Option> } } +const DUPLICATES_THRESHOLD: f32 = 0.95; + fn read_pq_codes(id: u32, index: Rc, buf: &mut Vec) { let loc = (id as usize) * index.pq_code_size; buf.extend(&index.pq_codes[loc..loc+index.pq_code_size]) } +struct VisitedNode { + image_url: String, + scores: Vec, + shards: Vec, + id: u32, + score: i64, + timestamp: u64, + dimensions: (u32, u32) +} + struct Scratch { visited_adjacent: HashSet, visited: HashSet, neighbour_buffer: NeighbourBuffer, neighbour_pre_buffer: Vec, - visited_list: Vec<(u32, i64, String, Vec, Vec)> + visited_list: Vec, + visited_embeddings: Vec } struct Index { @@ -140,10 +162,20 @@ async fn greedy_search<'a>(scratch: &mut Scratch, start: u32, query: &[f16], que let index = index.clone(); let node = handle.await?; let vector = bytemuck::cast_slice(&node.vector); - let distance = fast_dot_noprefetch(query, &vector); + let mut score = fast_dot_noprefetch(query, &vector); + score += descriptor_product(index.clone(), &descriptor_scales, node.id); cmps += 1; - if scratch.visited.insert(node.id) { - scratch.visited_list.push((node.id, distance, node.url, node.shards, node.scores)); + if scratch.visited.insert(node.id) && node.url.len() > 0 { + scratch.visited_list.push(VisitedNode { + image_url: node.url, + scores: node.scores, + shards: node.shards, + id: node.id, + score, + timestamp: node.timestamp, + dimensions: node.dimensions + }); + scratch.visited_embeddings.extend(bytemuck::cast_slice(&node.vector).iter().map(|x: &f16| x.to_f32())); }; for &neighbour in node.vertices.iter() { if scratch.visited_adjacent.insert(neighbour) { @@ -250,7 +282,8 @@ async fn evaluate(args: &CLIArguments, index: Rc) -> Result<()> { neighbour_buffer: NeighbourBuffer::new(args.search_list_size.unwrap_or(1000)), neighbour_pre_buffer: Vec::new(), visited_list: Vec::new(), - visited_adjacent: HashSet::new() + visited_adjacent: HashSet::new(), + visited_embeddings: Vec::new() }; let descriptor_scales = DescriptorScales(vec![0.0, 0.0, 0.0, 0.0]); @@ -265,14 +298,14 @@ async fn evaluate(args: &CLIArguments, index: Rc) -> Result<()> { println!("index scan {}: {:?} cmps", shard, cmps_result); } - scratch.visited_list.sort_by_key(|x| -x.1); - for (i, (id, distance, url, shards, scores)) in scratch.visited_list.iter().take(20).enumerate() { - let found_id = match matches.binary_search(&(*id, 0)) { + scratch.visited_list.sort_by_key(|x| -x.score); + for (i, node) in scratch.visited_list.iter().take(20).enumerate() { + let found_id = match matches.binary_search(&(node.id, 0)) { Ok(pos) => pos, Err(pos) => pos }; if args.verbose { - println!("index scan: {} {} {} {:?} {:?}; rank {}", id, distance, url, shards, scores, matches[found_id].1 + 1); + println!("index scan: {} {} {} {:?} {:?}; rank {}", node.id, node.score, node.image_url, node.shards, node.scores, matches[found_id].1 + 1); }; top_ranks[i] = std::cmp::min(top_ranks[i], matches[found_id].1); } @@ -341,11 +374,23 @@ pub async fn query_clip_server(base_url: &str, path: &str, data: Option Ok(result) } +#[derive(Serialize, Deserialize)] +struct TelemetryMessage { + #[serde(rename="correlationId")] + correlation_id: String, + data: serde_json::Value, + event: String, + #[serde(rename="instanceId")] + instance_id: String, + page: String +} + #[derive(Clone)] struct Service { index: Rc, inference_server_config: Rc, - args: Rc + config: Rc, + telemetry_channel: std::sync::mpsc::Sender } impl hyper::service::Service> for Service { @@ -355,15 +400,16 @@ impl hyper::service::Service> for Service { fn call(&self, req: Request) -> Self::Future { let index = self.index.clone(); - let args = self.args.clone(); + let config = self.config.clone(); let inference_server_config = self.inference_server_config.clone(); + let channel = self.telemetry_channel.clone(); Box::pin(async move { let mut body = match (req.method(), req.uri().path()) { (&Method::GET, "/") => Response::new(Full::new(Bytes::from(serde_json::to_vec(&FrontendInit { n_total: (index.header.count - index.header.dead_count) as u64, d_emb: index.header.quantizer.n_dims, - predefined_embedding_names: vec![] + predefined_embedding_names: config.descriptor_names.clone() })?))), (&Method::POST, "/") => { let upper = req.body().size_hint().upper().unwrap_or(u64::MAX); @@ -381,7 +427,7 @@ impl hyper::service::Service> for Service { &body.terms, &*inference_server_config, |batch, _config| { - query_clip_server(args.clip_server.as_ref().unwrap(), "/", Some(batch)) + query_clip_server(config.clip_server.as_str(), "/", Some(batch)) }, |image, config| async move { let image = image::load_from_memory(&image)?; @@ -397,27 +443,89 @@ impl hyper::service::Service> for Service { }).unwrap(); let selected_start = index.header.shards[selected_shard].1; - let beamwidth = 3; + let beamwidth = config.beam_width; let mut scratch = Scratch { visited: HashSet::new(), - neighbour_buffer: NeighbourBuffer::new(args.search_list_size.unwrap_or(1000)), + neighbour_buffer: NeighbourBuffer::new(config.search_list), neighbour_pre_buffer: Vec::new(), visited_list: Vec::new(), - visited_adjacent: HashSet::new() + visited_adjacent: HashSet::new(), + visited_embeddings: Vec::new() }; - let descriptor_scales = DescriptorScales(vec![0.0, 0.0, 0.0, 0.0]); + let mut desc = vec![0.0, 0.0, 0.0, 0.0]; + + for term in &body.terms { + if let Some(name) = &term.predefined_embedding { + if let Some(index) = config.descriptor_names.iter().position(|x| x == name) { + desc[index] = term.weight.unwrap_or(1.0) * 1.0/512.0; + } + } + } + + let descriptor_scales = DescriptorScales(desc); let query_preprocessed = index.header.quantizer.preprocess_query(&query); let query = query.iter().map(|x| half::f16::from_f32(*x)).collect::>(); - let cmps_result = greedy_search(&mut scratch, selected_start, &query, &query_preprocessed, &descriptor_scales, index.clone(), args.disable_pq, beamwidth).await?; + let cmps_result = greedy_search(&mut scratch, selected_start, &query, &query_preprocessed, &descriptor_scales, index.clone(), false, beamwidth).await?; - scratch.visited_list.sort_by_key(|x| -x.1); + let n_visited = scratch.visited_list.len(); - let matches = scratch.visited_list.drain(..).map(|(id, score, url, shards, scores)| (score as f32, url, String::new(), 0, None)).collect::>(); + let mut similarities_against_self = vec![0.0f32; n_visited * n_visited]; + + // runtime deduplicate of results list + unsafe { + // vecs @ vecs.T + matrixmultiply::sgemm( + n_visited, + index.header.quantizer.n_dims, + n_visited, + 1.0, + scratch.visited_embeddings.as_ptr(), + index.header.quantizer.n_dims as isize, + 1, + scratch.visited_embeddings.as_ptr(), + 1, + index.header.quantizer.n_dims as isize, + 0.0, + similarities_against_self.as_mut_ptr(), + n_visited as isize, + 1 + ); + } + + // discard anything similar to something already in list + let mut i = 0; + let mut included = bitvec::bitvec![0; n_visited]; + scratch.visited_list.retain(|_node| { + let row = &similarities_against_self[(i * n_visited)..((i + 1) * n_visited)]; + let old_i = i; + i += 1; + for (other_i, similarity) in row.iter().enumerate() { + if similarity > &DUPLICATES_THRESHOLD && included[other_i] { + return false; + } + } + included.set(old_i, true); + true + }); + + scratch.visited_list.sort_unstable_by_key(|x| -x.score); + + let matches = scratch.visited_list + .drain(..) + .map(|node| { + let debug = if body.debug_enabled { + Some((node.scores, node.shards, node.timestamp)) + } else { + None + }; + ((node.score as f64 / SCALE_F64) as f32, node.image_url, String::new(), 0, Some(node.dimensions), debug) + }) + .collect::>(); let result = QueryResult { formats: vec![], @@ -438,6 +546,25 @@ impl hyper::service::Service> for Service { .header(hyper::header::CONTENT_TYPE, "text/plain; version=0.0.4") .body(Full::new(Bytes::from(buffer))).unwrap() }, + (&Method::POST, "/telemetry") => { + // TODO refactor + let upper = req.body().size_hint().upper().unwrap_or(u64::MAX); + if upper > 1000 { + let mut resp = Response::new(Full::new(Bytes::from("Body too big"))); + *resp.status_mut() = hyper::StatusCode::PAYLOAD_TOO_LARGE; + return Ok(resp); + } + + let whole_body = req.collect().await?.to_bytes(); + + let message = serde_json::from_slice::(&whole_body)?; + + channel.send(message)?; + + Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Full::new(Bytes::from(""))).unwrap() + } (&Method::OPTIONS, "/") => { Response::builder() .status(StatusCode::NO_CONTENT) @@ -459,9 +586,9 @@ impl hyper::service::Service> for Service { } } -async fn get_backend_config(clip_server: &Option) -> Result { +async fn get_backend_config(clip_server: &String) -> Result { loop { - match query_clip_server(clip_server.as_ref().unwrap(), "/config", Option::<()>::None).await { + match query_clip_server(clip_server, "/config", Option::<()>::None).await { Ok(config) => return Ok(config), Err(err) => { tracing::warn!("waiting for clip server: {}", err); @@ -471,14 +598,31 @@ async fn get_backend_config(clip_server: &Option) -> Result, config: ServerConfig) -> Result<()> { + let mut telemetry_file = std::fs::OpenOptions::new().create(true).create(true).append(true).open(&config.telemetry_file)?; + while let Ok(message) = rx.recv() { + telemetry_file.write_all(rmp_serde::to_vec(&message)?.as_slice())?; + } + Ok(()) +} + async fn serve(args: &CLIArguments, index: Rc) -> Result<()> { + let config: ServerConfig = serde_json::from_slice(&std::fs::read(args.config_path.as_ref().unwrap())?)?; + + let (telemetry_channel, telemetry_receiver) = std::sync::mpsc::channel(); + + let config_ = config.clone(); + std::thread::spawn(move || telemetry_handler(telemetry_receiver, config_)); + let service = Service { index, - inference_server_config: Rc::new(get_backend_config(&args.clip_server).await?), - args: Rc::new(args.clone()) + inference_server_config: Rc::new(get_backend_config(&config.clip_server).await?), + config: Rc::new(config.clone()), + telemetry_channel }; - let listener = TcpListener::bind(args.listen_address.as_ref().unwrap())?; + let listener = TcpListener::bind(config.listen_address)?; println!("Listening"); loop { let (stream, _) = listener.accept().await?; @@ -531,7 +675,7 @@ async fn main() -> Result<()> { n_descriptors: header.descriptor_cdfs.len(), }); - if args.listen_address.is_some() { + if args.config_path.is_some() { serve(&args, index).await?; } else { evaluate(&args, index).await?; diff --git a/src/score_model.rs b/src/score_model.rs index 1be0d92..ed8fc10 100644 --- a/src/score_model.rs +++ b/src/score_model.rs @@ -53,8 +53,8 @@ impl ScoreModel { fn run_test_vectors() { let score_model = ScoreModel::load("model.safetensors", None).unwrap(); println!("load complete"); - let input = vec![-0.4345722794532776, 0.17094942927360535, -1.8118499517440796, -0.007176402490586042, -0.8931557536125183, 0.047411322593688965, 0.14499570429325104, 1.5815279483795166, -0.5733006596565247, 0.09284629672765732, -0.06355082988739014, 1.1163427829742432, 0.0919690877199173, -0.7623572945594788, -0.31675365567207336, -0.23023580014705658, -1.383502721786499, -0.022929714992642403, -0.4546741843223572, 0.4752075970172882, 0.0017476415960118175, 2.4422049522399902, 1.1863856315612793, -1.1715986728668213, 1.7562607526779175, 0.5107415914535522, 0.10338598489761353, 0.5254573225975037, 0.43233147263526917, 0.4230467677116394, -0.1960270255804062, -1.4768967628479004, 0.05825695022940636, 0.48162776231765747, 0.039969947189092636, -0.7935997843742371, -1.661239743232727, -0.3166193664073944, -0.44169801473617554, -0.8111487627029419, -0.008084080182015896, 0.01621660403907299, -0.21380890905857086, 1.2548106908798218, 2.5776288509368896, 0.9675433039665222, 1.0331544876098633, -1.276376724243164, -0.1208878681063652, 2.0618014335632324, -0.2110261768102646, 1.4121814966201782, -0.3874514102935791, 0.019096245989203453, -0.6084364652633667, -1.5250674486160278, 0.05467425659298897, 0.5211672186851501, -0.6599328517913818, 1.9770493507385254, -0.2652193605899811, -0.4438127279281616, -0.4085737466812134, -1.8672881126403809, -0.5800049901008606, -1.136744737625122, 0.6716499328613281, 0.02385501191020012, -1.4679572582244873, -0.20788317918777466, 1.78951096534729, -0.49180176854133606, -0.5267285704612732, -1.061163306236267, -1.167036771774292, -0.21071378886699677, 0.2671426236629486, 0.15262508392333984, -0.1835470199584961, 0.6520016193389893, -0.19917918741703033, 0.6172112822532654, -1.5099269151687622, 0.30058932304382324, -0.2920701205730438, -1.5939565896987915, 1.455906629562378, -0.5045331120491028, 1.087783694267273, 1.2643043994903564, 0.4651171863079071, -0.2950178384780884, -0.49719011783599854, -0.10433711111545563, 0.6476410627365112, -1.1793053150177002, -0.14007428288459778, 0.5063872933387756, -1.446657419204712, 0.4581936299800873, -0.14212673902511597, 1.6025649309158325, 2.506469488143921, -0.42433634400367737, 2.517164468765259, -0.5849226713180542, -0.5297185182571411, 1.024099349975586, -1.6314244270324707, -0.27694639563560486, -0.7040983438491821, -2.9417502880096436, 0.3231101930141449, 0.6132413744926453, -1.061632752418518, -0.6692830324172974, -0.16646189987659454, -1.5329070091247559, 1.2252086400985718, -0.6959750652313232, 0.6744255423545837, -1.2261444330215454, 0.5366888642311096, 1.8106290102005005, 0.8228190541267395, -0.44295385479927063, 2.1933095455169678, -1.2504291534423828, 0.8774024844169617, 0.8459886312484741, 0.9037753939628601, -0.4263146221637726, 1.7555049657821655, 2.5195908546447754, 0.16633424162864685, 0.9573475122451782, 0.8704742789268494, -0.589751124382019, -0.47897589206695557, 0.3424782454967499, -0.6239702701568604, -0.5562736392021179, -1.5022681951522827, 0.7230113744735718, -0.7862212657928467, 1.1114501953125, -0.4481828212738037, 0.9360501170158386, 0.18324320018291473, 1.6788688898086548, -0.004912206903100014, -0.32701143622398376, -0.08141219615936279, 0.6935409307479858, 1.5392712354660034, 0.42179566621780396, 0.18439936637878418, -1.3557062149047852, 1.906175136566162, 1.7605435848236084, 1.1143693923950195, 0.12973885238170624, -2.281808853149414, -0.7457895278930664, 0.563697874546051, -0.8167999386787415, 0.8724538087844849, -0.38157960772514343, 0.12257500737905502, 0.5205370187759399, 0.43817195296287537, 1.3469438552856445, 1.204759955406189, -0.812122642993927, 0.7565246820449829, -0.5746178030967712, 0.3362397253513336, 0.09120666235685349, -1.4427778720855713, -0.24839729070663452, 1.4323856830596924, 1.1117748022079468, -0.8627399206161499, -2.137739658355713, -0.0034492758568376303, -0.8460664749145508, -1.0586605072021484, 1.2009111642837524, 0.9322578310966492, -0.8685980439186096, -1.5788501501083374, 0.23623451590538025, -0.8549344539642334, 0.6803327202796936, 0.8026608228683472, -0.2921826243400574, -1.3224090337753296, 0.1882103681564331, -0.10856007784605026, 0.8612928986549377, -1.4223358631134033, 0.46004369854927063, -1.517471432685852, -1.220023274421692, 1.6014193296432495, 0.618366003036499, -0.4121616780757904, 0.9146530628204346, -1.2374593019485474, -0.014170434325933456, -0.3589692711830139, -0.6065499782562256, -0.20429876446723938, -0.9320686459541321, 1.226494550704956, 1.8877196311950684, -0.1909264624118805, -0.7644696235656738, -1.192503809928894, 0.5668321251869202, -0.6503121256828308, 0.005412652622908354, 0.6003715991973877, -0.16750405728816986, 0.730631411075592, -0.8260015249252319, -0.5752900242805481, -1.7150777578353882, -0.9912893772125244, -0.21429193019866943, -0.1821570098400116, 0.24360182881355286, 0.6649930477142334, -0.3617568910121918, 1.248687982559204, -0.16935500502586365, -0.27311739325523376, 0.19940800964832306, 1.211714506149292, -2.0409858226776123, -1.7912143468856812, 1.3380396366119385, 0.30974122881889343, -0.5215043425559998, 1.4406282901763916, 0.06237594783306122, -1.1279041767120361, 0.07083488255739212, -0.9160134792327881, 0.2535456120967865, -0.27451151609420776, 2.329385757446289, -0.6204302906990051, 0.7540424466133118, 0.25094467401504517, 1.107723355293274, -0.6849756240844727, 0.4012075662612915, 0.005810686852782965, -0.3159245550632477, 1.0356391668319702, 0.4598691761493683, -1.6891863346099854, -0.08487194031476974, 0.3393251299858093, 1.2522573471069336, -0.9809004664421082, 1.2244173288345337, -0.32187044620513916, 0.21259519457817078, -1.5967507362365723, 0.7714967727661133, 0.30796951055526733, -0.4697326719760895, -0.3248938322067261, -0.48808130621910095, 0.22755451500415802, 1.406537413597107, 1.0756304264068604, 0.03567849472165108, -1.4543119668960571, 1.2627283334732056, 0.3750774562358856, -0.6114221215248108, -1.208338975906372, -0.1788187325000763, -0.6459327936172485, -0.4125029444694519, -0.6384430527687073, 0.004708736203610897, 0.8536922931671143, -0.5542461276054382, -0.9046662449836731, 1.1861681938171387, -0.8095163702964783, 0.2388121336698532, 0.5717242956161499, -0.021248584613204002, -0.6682467460632324, 1.1803152561187744, 0.9752140045166016, 0.995647132396698, 1.7448264360427856, -0.7038049101829529, -0.5737085342407227, -0.4709729552268982, 1.0187917947769165, 1.1697276830673218, -0.06843794882297516, 1.3777273893356323, 1.4236022233963013, -0.06521105766296387, -0.9240316152572632, 0.24471965432167053, 2.1488571166992188, 0.23116901516914368, 1.0827425718307495, -0.09670041501522064, 0.2925485372543335, 1.0832446813583374, 0.04315602406859398, 0.3326874375343323, -1.770923376083374, -0.9208036065101624, -0.42884746193885803, 0.8803163170814514, 0.7154956459999084, -0.23689228296279907, -0.9407936930656433, -0.5876491069793701, 0.6052936315536499, 0.052228499203920364, 2.17978835105896, 0.16457822918891907, 1.1496567726135254, -1.803113341331482, -0.33117616176605225, -1.2849164009094238, 1.0336734056472778, -0.4086662530899048, 1.1697443723678589, -0.6452281475067139, -0.3375767171382904, 0.5617665648460388, 0.916449785232544, -0.41032272577285767, 0.6886813044548035, 0.40038833022117615, -0.5050271153450012, 0.11471790820360184, 0.1910623461008072, 0.22456428408622742, -2.027451515197754, 0.04354136809706688, 1.1241633892059326, -0.8220033049583435, -0.27855390310287476, 0.2972124218940735, 1.8156293630599976, 1.715639352798462, 0.06885554641485214, 0.38357144594192505, -0.08718876540660858, 1.7863292694091797, -0.6194261312484741, 0.8481863737106323, 0.5368261933326721, 0.6141453981399536, -0.8199034929275513, -0.31737208366394043, -0.11267230659723282, -0.5211840271949768, -1.0127308368682861, -1.5139232873916626, 1.1095565557479858, 0.7923423647880554, -1.210640788078308, 1.401814341545105, 1.361802339553833, 0.8248207569122314, -0.00010222350829280913, -0.6005297303199768, 1.007662296295166, 0.6106851100921631, 0.9532113671302795, 0.6760901212692261, 2.2247812747955322, -0.4495505690574646, 0.5501114130020142, -1.069810390472412, 0.6187455058097839, 0.7255762815475464, -0.06195748597383499, 1.5735702514648438, -1.6713207960128784, -0.2957589626312256, 1.752336859703064, -0.9450791478157043, 0.08109347522258759, -1.7591547966003418, 0.9438595771789551, 1.034585952758789, -0.30059513449668884, -0.933298647403717, -0.3617241084575653, 0.03835463896393776, 0.5327099561691284, 0.7001193165779114, 0.7219358682632446, -2.126343011856079, 1.3780051469802856, 0.6344947814941406, 0.5614579319953918, -1.413107991218567, 0.09966818988323212, -0.4681704640388489, 0.060804836452007294, 0.9871187210083008, 1.8834701776504517, 0.7778974175453186, 0.007839275524020195, 0.5914214849472046, 0.9574611783027649, 0.4493805766105652, -1.6901856660842896, -0.979607105255127, 1.2487770318984985, -0.8207526803016663, -2.2255351543426514, 0.5124965310096741, 0.6362564563751221, -0.46572402119636536, -0.13918595016002655, -0.6213287115097046, -0.570345938205719, 0.19373509287834167, 0.3811667859554291, 0.26523342728614807, -1.084452509880066, -1.267290472984314, -1.5294244289398193, 1.0089960098266602, -0.2576083242893219, 1.0372828245162964, -2.597663640975952, -0.450050413608551, 0.6596473455429077, 0.3285040259361267, 0.47114452719688416, 1.4923033714294434, -0.5632882714271545, -0.7333149909973145, -0.8307684659957886, -1.5851761102676392, -0.25743192434310913, -0.36466726660728455, 0.08711828291416168, 0.2192882001399994, -0.3982499837875366, -1.304565668106079, 0.09079232811927795, -1.6411323547363281, -0.5723531246185303, -0.9254944920539856, 0.5284976363182068, -0.9033693075180054, -0.13852863013744354, -0.6156182289123535, -0.549788236618042, -1.5759395360946655, -1.294786810874939, -0.28782978653907776, -0.8077318072319031, 0.029211968183517456, 2.404716968536377, 0.3512919843196869, 0.3920923173427582, -0.5422165393829346, -0.5234830975532532, 0.9085437655448914, -0.8280503153800964, -1.061059832572937, 0.12982042133808136, 0.8198115229606628, 0.18910229206085205, -0.7295379042625427, 0.31590694189071655, 1.009547472000122, -1.1249982118606567, 0.4857461452484131, 0.4465593099594116, 0.8031508922576904, 1.9976060390472412, -0.5282385945320129, 1.0601937770843506, -0.11239448189735413, -0.6282514929771423, -0.2416393756866455, 0.13363239169120789, 0.47433510422706604, -1.0977619886398315, -0.2604287564754486, 0.09573862701654434, 0.2108738273382187, 0.5167774558067322, -0.6554499864578247, -0.5711472034454346, 1.381605863571167, -0.6697666645050049, -0.15287402272224426, -0.8433055877685547, -0.5154555439949036, -0.08705288171768188, 0.21231800317764282, 1.0561351776123047, 0.3892776668071747, 1.502345085144043, -0.6005531549453735, 0.9926665425300598, 0.44098135828971863, 0.6222519874572754, 0.9260947108268738, 1.3715100288391113, 0.2827765643596649, -0.9424709677696228, -1.3695074319839478, -0.4716727137565613, -0.07552099972963333, -0.6941318511962891, -0.1219126284122467, 0.6719748377799988, 0.25753113627433777, -1.2153089046478271, 0.13151207566261292, -0.6439849138259888, 0.5595400929450989, -0.09089384227991104, 0.6474713683128357, 0.10356837511062622, -0.06700046360492706, -0.0591135174036026, -0.38273537158966064, 1.0885146856307983, -0.6373236179351807, 1.1865304708480835, -0.7767115831375122, -0.8741889595985413, -0.2947963774204254, 0.677485466003418, 0.2559397518634796, -1.9060794115066528, 2.800122022628784, -0.16625577211380005, 0.8319082260131836, -0.04343127831816673, 0.03674604371190071, 0.686943531036377, 1.988837480545044, 0.1322024166584015, 1.63629949092865, -0.6030665636062622, 0.9368968605995178, 0.05773721635341644, 0.057696983218193054, -0.6373687386512756, -1.0348992347717285, 2.306628942489624, 1.791831612586975, 0.9562671184539795, -0.7249517440795898, -0.013636152260005474, -1.0798742771148682, -0.6930895447731018, -1.519718885421753, -0.31140774488449097, 0.7926839590072632, 0.15577048063278198, 0.582065761089325, -1.1615068912506104, 1.2250007390975952, -0.4072926342487335, -1.526711344718933, 0.8438209891319275, 0.5494145750999451, 1.9605902433395386, -0.8468219637870789, -0.7103831171989441, -0.6975318193435669, 0.9697279334068298, 0.9154353737831116, -1.3216902017593384, -0.2999522089958191, -0.709892749786377, 0.23295240104198456, 0.17478814721107483, -0.5722963213920593, 0.3553563356399536, 0.4296479821205139, 1.382393717765808, -0.26311346888542175, 0.664761483669281, 0.8819129467010498, -0.6056832075119019, -1.0869754552841187, -1.4896304607391357, -0.005215405486524105, 1.6510801315307617, -0.9862425923347473, 1.8971830606460571, -0.7558636665344238, 2.0353755950927734, -0.9452323913574219, -1.3971489667892456, -0.5639879703521729, -0.7930155992507935, 0.0170948076993227, -0.07290758937597275, -0.5839744806289673, 0.05692382529377937, -0.14844456315040588, -0.18833090364933014, -0.4581572413444519, 0.645013689994812, 0.13673590123653412, 1.7292461395263672, -0.9524289965629578, -2.997218370437622, 0.27123352885246277, 0.6761008501052856, -0.16884158551692963, 0.968781054019928, -1.9877904653549194, -0.2800651788711548, -0.6947948336601257, 0.23747152090072632, 0.9826598167419434, 1.6584845781326294, 0.7255685329437256, -0.33844730257987976, -0.7704879641532898, 1.4175623655319214, 1.3734723329544067, -2.0814011096954346, 0.6412269473075867, -0.33805736899375916, -0.046853456646203995, 0.359149694442749, 0.39657217264175415, -0.5688016414642334, -2.538172483444214, 0.3339379131793976, -1.0108493566513062, 0.6112464666366577, -0.7258294224739075, 0.3515496850013733, -0.4045449495315552, 0.673316478729248, -1.7248252630233765, -0.7068468332290649, 0.057085003703832626, 0.38993415236473083, 0.7348064184188843, -0.3441391885280609, 0.5581517815589905, 1.1088072061538696, 0.23822224140167236, -0.5095551013946533, -2.6404576301574707, -0.6723653674125671, 0.8111631274223328, -0.061103880405426025, 0.652798056602478, 0.5370320081710815, 0.9253896474838257, 0.054714102298021317, 0.6374782919883728, 0.7382916212081909, 0.976263165473938, -0.22624091804027557, 0.5979400873184204, 0.8526683449745178, -0.1370960921049118, -0.5161216259002686, 0.579807698726654, -0.7243971824645996, 0.21032129228115082, -0.44252756237983704, 0.9881635904312134, 0.2994612455368042, -1.0705238580703735, -0.3441469371318817, 1.0209137201309204, 0.4681636095046997, 0.509205162525177, 0.6201439499855042, -0.6139274835586548, 1.7681114673614502, 0.05952373519539833, 0.44398602843284607, -0.21524061262607574, 0.6050172448158264, 0.7010953426361084, -0.5873063206672668, -0.18935400247573853, -0.4448927342891693, -1.0313451290130615, -0.5306463837623596, -0.22999121248722076, 1.0428887605667114, 1.2442805767059326, -1.1165685653686523, -1.5585505962371826, 0.504208505153656, 0.5706216096878052, -0.6087542176246643, 0.4025251567363739, -0.33637118339538574, 0.5983011722564697, 0.3604276776313782, 0.1312180906534195, -1.4141628742218018, -1.6563067436218262, 1.9902223348617554, -0.6246545314788818, -0.7443554997444153, -1.7187235355377197, -1.594356894493103, 1.5916907787322998, -0.659320056438446, 0.9399675130844116, 0.3772321343421936, 1.176887035369873, 0.5420020222663879, -0.33470937609672546, 0.12828439474105835, -0.001921453163959086, -0.060976818203926086, -0.6981583833694458, 1.3173587322235107, 0.3819849193096161, -1.469565749168396, 1.198104977607727, 0.04355635493993759, 1.0236657857894897, -1.0002162456512451, -0.5398812890052795, 0.008763191290199757, 0.38617855310440063, 1.907233476638794, -0.38517826795578003, 0.4452987313270569, -0.2268972098827362, 1.5880435705184937, 0.6075268387794495, 0.5430085062980652, 0.5517557263374329, -0.23936240375041962, 0.362492173910141, 0.06497232615947723, 0.41428202390670776, 1.998616337776184, 0.54366135597229, 1.0919013023376465, -0.6124905943870544, 1.061617136001587, -1.2426657676696777, -0.8263672590255737, -0.8436780571937561, 0.8996731638908386, 1.4643795490264893, -1.0802556276321411, -0.4260358512401581, 3.515040397644043, -0.5122134685516357, 0.6618409752845764, 0.7780985236167908, 0.5200847387313843, -1.407347321510315, -0.6305193901062012, 0.6847864389419556, 0.2808320224285126, -0.37852010130882263, 1.4333117008209229, 0.8095510005950928, 0.34315812587738037, -1.1373587846755981, -0.2777940034866333, 1.097699522972107, 1.055644154548645, -1.1646078824996948, -1.293685793876648, 0.14235921204090118, -0.45536214113235474, -0.7593684196472168, 1.645435094833374, -0.45230573415756226, -0.06950368732213974, -0.9636492729187012, 0.02589770033955574, 1.2381441593170166, 0.07427545636892319, 0.4933584928512573, -0.25906309485435486, 0.42474111914634705, -0.022335702553391457, 0.3286267817020416, -0.6273146867752075, 0.7566981911659241, -1.6992632150650024, 1.199073314666748, 0.2219061404466629, 0.4499109089374542, 2.7969255447387695, 0.2582072615623474, -0.03357900679111481, 1.6473480463027954, -1.176878809928894, 0.6138300895690918, -1.367021083831787, 0.07303396612405777, -0.8029521107673645, -0.09143408387899399, -0.06039535254240036, 0.004455065820366144, 0.3565599024295807, 1.528414011001587, 1.5721116065979004, 0.9596143364906311, -1.220434546470642, 0.5388507843017578, -1.119903802871704, -0.5057213306427002, -1.048045039176941, 0.03703024983406067, -1.0204658508300781, -1.2635310888290405, -1.1467036008834839, -0.7836745977401733, 1.576639175415039, -1.3404783010482788, 1.4718537330627441, 0.025984615087509155, -1.3582971096038818, 0.027891667559742928, -0.26313361525535583, 0.20543237030506134, 0.23938529193401337, 2.1292598247528076, 1.1284244060516357, -1.2642018795013428, -0.11442836374044418, -0.39886006712913513, -0.23897622525691986, 1.0716161727905273, 0.17985078692436218, -1.8444756269454956, 0.6462835669517517, 0.47688910365104675, 0.04440760239958763, 0.3059315085411072, -0.3850139081478119, -1.419514536857605, 0.4204521179199219, 0.12816652655601501, 0.6466811895370483, -0.25078675150871277, -0.17331160604953766, 0.14207015931606293, -0.5685902833938599, -1.519852638244629, 0.4267028272151947, 0.013892977498471737, 0.5958963632583618, 1.7657804489135742, -1.0036885738372803, -1.605370283126831, 0.7875543832778931, 0.4335712492465973, 0.7983883023262024, 0.0908045619726181, 0.5869224071502686, -0.6014065146446228, 0.12830570340156555, 0.6985166668891907, 0.0337148942053318, 1.6846498250961304, -0.29135188460350037, 0.32195645570755005, -1.0164788961410522, 0.19770798087120056, -0.7525082230567932, -2.210667848587036, 0.5082350969314575, -0.12018764764070511, 0.2551785707473755, 0.13795655965805054, 1.7685922384262085, 0.21039587259292603, -0.838496208190918, -0.09064681828022003, 1.0909736156463623, -1.1924850940704346, 0.8824757933616638, 0.9496980905532837, -1.7810113430023193, -0.4604567587375641, -1.0911777019500732, -0.38226792216300964, -0.02809564210474491, 0.786867082118988, -0.3252718448638916, -0.4239846169948578, 0.6905282139778137, -0.5622822642326355, 0.7191290259361267, -0.7628090381622314, -0.1824864149093628, 0.2172347456216812, 1.0432536602020264, 0.6175377368927002, 1.181478500366211, -0.13423621654510498, 1.5229978561401367, 0.5425287485122681, -1.0304006338119507, 0.4910985231399536, 0.22717326879501343, -0.28802478313446045, 1.371718406677246, -0.08901949226856232, 0.313540518283844, -1.2450536489486694, 0.5087664127349854, -0.07295728474855423, 0.7783309817314148, 0.5604421496391296, 0.05062832683324814, -2.2363927364349365, 1.3607529401779175, 0.32394710183143616, 1.3631151914596558, 0.6177082657814026, 0.554364025592804, 1.2375513315200806, 0.9964534640312195, -1.2423371076583862, -0.8665270805358887, -0.030565202236175537, 0.13943910598754883, 0.6958285570144653, 0.4483960270881653, -0.8183225393295288, 1.5288769006729126, -0.47511765360832214, 1.1311359405517578, -1.621194839477539, 0.8746280670166016, -0.8607368469238281, 0.7878744602203369, 0.9095972776412964, 0.652429461479187, 0.7531909346580505, 0.3955795466899872, 0.8608068823814392, 0.49420660734176636, -0.18280136585235596, -2.3184430599212646, 0.7269343733787537, -0.32976558804512024, -0.616654098033905, 1.1827670335769653, 0.8830099701881409, 0.08306678384542465, 1.198328971862793, 0.8751806020736694, 1.3158116340637207, -2.0485854148864746, 0.024729225784540176, -1.2841745615005493, -0.8687040209770203, 0.8172692656517029, 1.2917695045471191, -0.3189697563648224, 1.0904253721237183, 0.11204420775175095, -0.49574729800224304, 0.49854835867881775, 0.30887171626091003, 0.648361086845398, -0.9476398229598999, 0.14023321866989136, 1.485676646232605, -0.11024042218923569, -0.46792498230934143, -1.0652883052825928, -1.7874857187271118, -0.3664189279079437, -0.46078261733055115, -0.7502343058586121, -0.2235097438097, 0.8470032811164856, 2.1356754302978516, 1.0604249238967896, 0.1708608716726303, 0.022037718445062637, 1.2777403593063354, 1.0032052993774414, 0.19882464408874512, -0.587616503238678, -0.37336447834968567, 2.0249993801116943, 1.4226454496383667, -0.3835103213787079, -0.5216581225395203, 0.3705710768699646, -1.3748290538787842, -0.09237286448478699, -0.18686658143997192, 0.09567052125930786, 0.2889290750026703, 0.950859546661377, 0.360861212015152, 0.09397242218255997, 0.7437982559204102, -0.49909383058547974, 0.4474595785140991, -0.3323509991168976, -1.0463372468948364, 0.048776689916849136, 0.9524378776550293, -0.09515365958213806, -0.09430793672800064, 0.14550821483135223, 1.1051427125930786, -0.8149166703224182, -0.46799570322036743, -0.6497594714164734, 1.290147066116333, -1.3319333791732788, 0.8443806171417236, 0.6181923151016235, -0.6171462535858154, -0.7310259342193604, -0.5035132765769958, -0.00936672743409872, 0.35036027431488037, 1.3523614406585693, 0.09188584238290787, 1.0885218381881714, 0.4177442491054535, -0.6786648035049438, -0.06243576854467392, -0.7687265276908875, 0.42035430669784546, 0.03898458555340767, -0.6579251885414124, -0.40868568420410156, -0.5622940063476562, -1.1191990375518799, -0.4757128357887268, 1.2259920835494995, -0.2573641836643219, 1.1551967859268188, -0.1616869568824768, -0.9229593873023987, 0.17305219173431396, 0.48502930998802185, 0.012383662164211273, 0.0564519539475441, 0.6684853434562683, -0.4846735894680023, -0.1267959475517273, 0.003160330932587385, -0.41201651096343994, 0.18376967310905457, 0.12061426788568497, 1.4958235025405884, -0.8688951730728149, -0.15029418468475342, -1.181771993637085, -2.026181221008301, 1.775275707244873, -1.364978313446045, -1.137622356414795, -1.0544930696487427, 1.3910095691680908, -1.1349668502807617, 1.9832195043563843, 1.0841591358184814, -0.34837594628334045, -1.2041701078414917, -0.5303089618682861, -1.1168203353881836, -0.994713306427002, -0.5010892748832703, -0.4602827727794647, -0.049978870898485184, -0.6047748327255249, -0.4769015312194824, 0.21973305940628052, -1.5256102085113525, -0.5643079876899719, -0.3512163758277893, 0.529365599155426, 0.43356889486312866, -0.6662517189979553, 0.4539867043495178, -0.8593372702598572, -1.773324728012085, -1.1821492910385132, 1.554931640625, 0.8177838921546936, -0.3897852301597595, -0.261726051568985, 0.3987899124622345, -0.2785939574241638, 0.4711616039276123, 1.502377986907959, -0.17331713438034058, -0.3258228003978729, 0.6905822157859802, 0.34060052037239075, 1.3842672109603882, -0.5461441874504089, -1.8935116529464722, 0.16089187562465668, -0.49302351474761963, -0.6302204132080078, 1.1954755783081055, 1.182828426361084, 0.5417712926864624, -0.3421955406665802, 0.4476862847805023, 0.09665082395076752, -0.77329021692276, 0.3875643014907837, 1.3439069986343384, -2.6292200088500977, 0.18065409362316132, 0.646568238735199, -1.5938225984573364, 1.5059627294540405, 1.0302311182022095, -1.5075165033340454, -0.6646416187286377, -0.5766743421554565, -0.7726435661315918, 0.26617905497550964, 1.509953260421753, -0.8543895483016968, 0.3991389572620392, 1.9726192951202393, 0.9037456512451172, 0.7837607264518738, -0.3070720136165619, -0.3035714030265808, -0.11029664427042007, 1.0072765350341797, -0.08948825299739838, -1.332390308380127, -1.3333803415298462, -0.13565726578235626, 0.23599256575107574, 0.5334976315498352, -0.38123127818107605, -0.5135204792022705, -1.0396808385849, -0.4158479869365692, 0.7913282513618469, 0.4868917763233185, -0.8251904845237732, 2.381834030151367, 1.1846328973770142, -0.32473024725914, -0.7890875339508057, 0.9400415420532227, -0.1604483723640442, -0.7967333197593689, 0.31567293405532837, 0.5375560522079468, 1.0065821409225464, -0.2816874384880066, 0.8914854526519775, -0.08214523643255234, 0.6124990582466125, 1.70603609085083, -1.1971832513809204, 1.7796889543533325, -0.09240670502185822, -0.4935224652290344, 0.1219179555773735, 0.47707226872444153, 0.12535817921161652, -0.10471872240304947, -0.7864277958869934, 0.5792407989501953, 2.305544137954712, -1.58732271194458, -1.6944420337677002, -0.44738689064979553, -0.7232373952865601, 0.30735182762145996, -1.6345329284667969, -0.2897382974624634, -0.4674256145954132, -2.1037044525146484, 0.7550560235977173, 1.5686960220336914, 0.47492778301239014, 1.3041026592254639, 0.2556685209274292, 0.8898839950561523, -0.9076237082481384, 0.4316483736038208, 0.43863430619239807, 0.510120689868927, 0.018206022679805756, 1.7730722427368164, 0.052153728902339935, 0.6367270946502686, -0.6456711292266846, -2.7352042198181152, 1.9065017700195312, 0.9640504717826843, 1.1189193725585938, -0.8896899223327637, 1.2418859004974365, -0.20937834680080414, 0.6801437735557556, 0.31543347239494324, 0.2170463502407074, -0.846274197101593, -0.12267930805683136, -0.17491790652275085, 0.09508680552244186, 0.8562034964561462, -0.06768842041492462, -0.28452029824256897, -0.9927764534950256, -0.2832501232624054, -0.10093846172094345, 0.9885483980178833, 1.2435678243637085, 0.6468109488487244, -0.9302873015403748, 0.15413260459899902, 0.9843354225158691, -1.0882291793823242, 0.6943358778953552, -2.121218681335449, -0.39891448616981506, 0.6712089776992798, -1.0961931943893433, -0.6278862953186035, -1.312329888343811, -1.6472008228302002, -0.25670745968818665, 0.5573739409446716, 0.1816536784172058, 1.9136946201324463, 0.05819267779588699, 1.2408310174942017, 0.9116583466529846, 1.23573899269104, 0.8580152988433838, 0.1878761649131775, -0.40929684042930603, -0.49410054087638855, 0.4195306599140167, -0.7070677876472473, -0.8550442457199097, 0.038285087794065475, -0.5699215531349182, 1.5516647100448608, -0.23938804864883423, 1.8143301010131836, 1.1729305982589722, -0.7128189206123352, 0.6277351379394531, 0.3338187038898468, -1.656206488609314, 0.6193100810050964, 1.5606101751327515, -1.2442196607589722, -0.7162578105926514, -0.8671603798866272, -0.3340846300125122, 0.14698074758052826, -0.4165472984313965, -0.06832127273082733, 0.4592666029930115, 0.46814027428627014, -0.5336564183235168, 1.0729955434799194, -0.5952112078666687, 1.21340012550354, 1.213293194770813, 0.04860914126038551, -0.3598390817642212, 1.1526411771774292, -0.6540412306785583, 1.619145393371582, -0.07900376617908478, 0.5354835391044617, -0.5690369606018066, 0.24075555801391602, 0.5562435388565063, -0.5513310432434082, 0.43661242723464966, -0.5632359981536865, -0.7913996577262878, -1.0616430044174194, 0.9400043487548828, 0.24701431393623352, -0.2364419847726822, -0.18591170012950897, -0.4300668239593506, -0.25861018896102905, -0.19917157292366028, 0.6145332455635071, 0.6251468062400818, 1.6782547235488892, 1.2751544713974, 0.017351359128952026, 0.11740890145301819, -0.5722630023956299, 0.6933882832527161, 1.0625139474868774, 1.9571846723556519, -0.23565895855426788, -0.5926447510719299, -0.7708045840263367, 0.6210179328918457, -0.13394542038440704, 0.5444365739822388, 0.5795621871948242, -0.613130509853363, -0.3769218325614929, 0.3578146696090698, 0.3179452121257782, -0.0924324095249176, 0.4720917344093323, -1.3465638160705566, 0.773696780204773, 1.1130074262619019, -3.880899429321289, 0.7001267075538635, -1.0525175333023071, 0.916896641254425, -1.4023064374923706, 0.373981773853302, 0.6901137232780457, -0.0072166817262768745, -0.20865239202976227, -0.889133632183075, -0.7716237306594849, 0.011725494638085365, -1.3447166681289673, 0.036912478506565094, -0.4225759506225586, 0.6905183792114258, -1.8914035558700562, 1.1470389366149902, 1.4775840044021606, 0.8099463582038879, -0.3924289345741272, -0.04542950913310051, 1.5653220415115356, -1.0930054187774658, -1.1775825023651123, -0.49021753668785095, 0.9882652759552002, 0.23169665038585663, -1.6516475677490234, -0.9598382711410522, 0.6492452025413513, 0.9127811193466187, 0.22018083930015564, 2.085073947906494, 2.081601858139038, 0.055029720067977905, 0.5874921083450317, 0.8527942299842834, 0.46423062682151794, 0.5131237506866455, 1.2076706886291504, 0.4814620614051819, -0.2141030877828598, 0.31744271516799927, 0.27498605847358704, -1.0714075565338135, -0.28000950813293457, 2.0789597034454346, 0.8551387786865234, 0.6866350173950195, -0.0068307542242109776, 1.104396939277649, -1.4655455350875854, -0.12386060506105423, -1.7545208930969238, 1.8487300872802734, 0.10338234156370163, -0.47586047649383545, 1.0276609659194946, -0.057969823479652405, 0.6120775938034058, 0.2864364981651306, -0.08156619220972061, -0.1420280933380127, -0.10340193659067154, 1.1370623111724854, -1.765679121017456, -1.1121169328689575, 0.9350041747093201, -1.326656699180603, 0.26055893301963806, 0.426167756319046, -1.6100777387619019, -0.5306416749954224, -0.4269423186779022, 1.4333020448684692, 0.5915703177452087, -1.2162346839904785, 2.054032564163208, -0.9123718738555908, 2.064749240875244, 1.0158973932266235, 2.081069231033325, 0.7514635324478149, 0.3657980263233185, 0.2907051146030426, -1.390509843826294, 0.1681639403104782, 0.26740849018096924, -2.3427581787109375, 0.9623771905899048, 0.2362469583749771, 1.6433788537979126, -1.7055786848068237, 0.06149706989526749, 2.090935468673706, 0.2196318805217743, -1.5268381834030151, 0.8437720537185669, -0.0021424090955406427, 1.714131236076355, 1.3084003925323486, -0.12800508737564087, 0.9908474087715149, -1.2213125228881836, -0.49883392453193665, 0.34899067878723145, -0.645725429058075, -0.42813172936439514, -1.1531367301940918, 0.5270371437072754, -1.315256118774414, 0.3413543999195099, -0.6871671080589294, 0.3409877419471741, 0.570871889591217, -1.8945289850234985, -0.8591678738594055, -0.7143080234527588, -0.5257779955863953, 1.047529697418213, 0.669776976108551, 0.2726799547672272, 1.5355489253997803, 0.39360541105270386, -1.4439979791641235, 0.01889875717461109, -1.3284087181091309, -1.6329928636550903, -0.219257190823555, -0.3570577800273895, 0.19290809333324432, 0.7005501389503479, -0.8751274347305298, 0.7017621994018555, -0.18943041563034058, 0.04669550061225891, -0.5396027565002441, -0.4151298999786377, -1.5333091020584106, 0.46353310346603394, 1.2695139646530151, 0.37849879264831543, -0.5271384119987488, -2.0322461128234863, 1.3986854553222656, -0.3808351159095764, 1.9051475524902344, -0.3201722502708435, 1.1553555727005005, 1.7925398349761963, -1.8154748678207397, -0.030836760997772217, -0.43297839164733887, -0.7873007655143738, -0.4279592037200928, 0.5663005113601685, -0.9894522428512573, 1.038629412651062, 1.2019994258880615, 0.35455480217933655, -0.5538424849510193, 0.4213135242462158, -1.5061092376708984, 0.6037986874580383, 0.2639113962650299, -0.6716237664222717, 2.1139564514160156, 0.10385753214359283, 0.038658902049064636, 0.9666663408279419, 0.30583876371383667, -0.5329084992408752, 1.2298007011413574, 0.4459278881549835, -0.23942025005817413, 0.4124913811683655, -0.07009147852659225, -0.04792704060673714, 0.4795272648334503, -0.2518480718135834, 1.5541616678237915, -0.1981630027294159, 0.9096081256866455, 0.1782306581735611, -0.7890022397041321, -0.002378856297582388, -0.09527285397052765, -0.14582587778568268, -0.5931705236434937, 0.34214574098587036, -0.2859426438808441, -2.43243145942688, -0.5971176028251648, -0.06615541130304337, 0.601423442363739, 1.5045653581619263, -0.03493335843086243, 1.264575481414795, 0.9818770289421082, 0.2262442708015442, -0.8509559631347656, 0.47027158737182617, -1.6728217601776123, -0.5056917667388916, -0.21781963109970093, 0.11739428341388702, -1.1350071430206299, 0.19094473123550415, 0.416875958442688, 0.3947962522506714, -0.832273542881012, -1.3818416595458984, -0.17807288467884064, 0.18109138309955597, 0.5079516768455505, 1.00491201877594, -1.2930830717086792, 0.0031197841744869947, 0.4002855122089386, -0.6698902249336243, -0.03138754889369011, -0.04883351922035217, -0.47307252883911133, 1.439782977104187, 0.9919193983078003, 0.9581447839736938, 1.7779579162597656, -0.15189355611801147, 1.9064451456069946, 0.8630735278129578, 0.4774859547615051, -1.6237406730651855, 1.3787522315979004, -0.6637470126152039, -1.5003209114074707, -0.832725465297699, 1.1324589252471924, -1.268568992614746, -0.10494667291641235, -2.0737740993499756, -1.5177022218704224, -0.12315963953733444, 0.8319310545921326, 0.07761915773153305, -0.2953285872936249, -0.09603111445903778, -2.0960352420806885, 0.42228788137435913, 0.6378287076950073, -0.5993842482566833, -0.48815691471099854, 1.2753726243972778, 0.6403452157974243, 0.136467844247818, 0.9466603398323059, -0.8685232996940613, -0.6331071257591248, 0.5138336420059204, -0.43248939514160156, 1.2271833419799805, -0.3379306495189667, -0.5109993815422058, -0.11944505572319031, 0.432911217212677,-0.4665447175502777, -1.6220837831497192, 0.36841392517089844, -0.4407062530517578, -2.0674679279327393, -0.6870763301849365, 2.528989553451538, -0.9203405976295471, -0.38019734621047974, 0.35331693291664124, 1.2777550220489502, -0.7886718511581421, -0.159354105591774, -0.6902609467506409, -0.8750301003456116, -0.06744105368852615, 0.003242728766053915, 0.4032633304595947, 1.195798397064209, 0.6869891881942749, 0.45075178146362305, -1.260483980178833, -2.225640296936035, -0.9323518872261047, -0.669847846031189, 0.6522054672241211, 0.040609318763017654, 0.6746787428855896, -1.0279103517532349, -1.5991724729537964, 1.697393774986267, -0.9397741556167603, -0.09880328178405762, 0.34120890498161316, 0.35589972138404846, -1.4798641204833984, 0.4242649972438812, -1.7535536289215088, 1.5481618642807007, 0.7435497045516968, -0.8196710348129272, -0.42060360312461853, -1.238357424736023, 0.4606473445892334, -0.21138827502727509, -0.6937668919563293, 1.2042243480682373, -0.43613678216934204, -1.8454593420028687, -1.058915138244629, -0.4711950719356537, -0.47125717997550964, 1.2740403413772583, -1.372563123703003, 0.07787882536649704, 0.7184062600135803, 1.7741420269012451, -0.7095621228218079, -0.6356222629547119, -0.12805472314357758, -0.031177060678601265, 0.47538813948631287, 1.2984262704849243, -2.4233570098876953, -0.48652705550193787, 0.3928956985473633, -0.28848138451576233, -0.9190113544464111, 1.273335576057434, -1.153786301612854, 0.4283009171485901, -0.9839879274368286, 1.8350701332092285, -0.7429067492485046, 0.3393177390098572, -0.708436131477356, -0.811260998249054, -0.40281379222869873, 1.793811559677124, -0.8954534530639648, 1.3534959554672241, -1.5749458074569702, -0.20664086937904358, -0.4277615249156952, -0.6265417337417603, -1.569759726524353, 1.0968217849731445, 1.0153733491897583, 1.0656358003616333, 0.349366158246994, 0.25450289249420166, 0.7679623365402222, -0.17604440450668335, 1.6993589401245117, -1.5282362699508667, 1.8526616096496582, -1.2533150911331177, -0.9407021999359131, -0.6378714442253113, 0.5630449652671814, 0.5502960681915283, -0.42558467388153076, 0.07344574481248856, -0.6200817227363586, 0.23960959911346436, 1.2764447927474976, -0.332558274269104, -0.10788929462432861, -0.9556770920753479, -0.11629515886306763, -0.9808346033096313, 0.9965885877609253, -0.05854010581970215, -0.029060829430818558, 0.33677077293395996, 0.8061453700065613, 0.11750955134630203, -0.25667044520378113, -0.15457609295845032, 0.24936281144618988, 0.22214218974113464, 0.2265612781047821, 0.49928930401802063, -0.1673116236925125, -1.7134121656417847, -1.1109263896942139, -2.2883825302124023, 1.2363616228103638, -1.251094102859497, -1.3024230003356934, -0.8496585488319397, 1.362679362297058, 1.2713948488235474, 0.5061668753623962, -0.02020910009741783, -0.664803147315979, 1.3355088233947754, -0.051033664494752884, 0.8295128345489502, -2.1890697479248047, 1.007643461227417, -0.014182155951857567, -1.6302287578582764, 0.8181455731391907, -0.5556927919387817, 0.7454536557197571, -0.804707407951355, -1.4275274276733398, 0.8486582040786743, -1.1276911497116089, 0.6683546900749207, 0.1383368819952011, -1.1100330352783203, 0.9642177820205688, 0.7522894144058228, -0.7745781540870667, 2.4359591007232666, -1.6414024829864502, 0.5103375911712646, 0.35149580240249634, 2.9079368114471436, 1.3248181343078613, -0.5102512836456299, 0.3427905738353729, -1.2181408405303955, 0.08050795644521713, 0.7328014373779297, 0.25450754165649414, -1.1155622005462646, -0.39821669459342957, 0.3699052035808563, 2.260359287261963, 1.1545430421829224, -0.17546917498111725, -1.7796928882598877, 0.5038288235664368, -0.05621930956840515, -0.8811786770820618, 1.215942621231079, 1.2839049100875854, -0.7224724292755127, 0.6547309756278992, -0.025524424389004707, 1.5243316888809204, 0.1444752961397171, 0.03916008025407791, -1.0334433317184448, 0.3629348874092102, -0.7248011827468872, 1.1726082563400269, 1.1112247705459595, 1.219386100769043, 0.36894917488098145, -1.3814780712127686, -1.6700208187103271, 0.1797226518392563, -0.8019726872444153, -0.19178606569766998, -0.33795371651649475, -0.18185321986675262, -1.6810351610183716, -0.885891854763031, -0.22507618367671967, -1.2536576986312866, -0.02621975541114807, 0.030909035354852676, 1.4307403564453125, 1.276822566986084, 0.13052316009998322, -1.074567437171936, -1.0886785984039307, -0.6965903043746948, 1.3606441020965576, -0.6863150596618652, -0.3978932797908783, -0.45347973704338074, -0.2479778230190277, 0.18238626420497894, 1.057168960571289, -1.3291025161743164, -0.6565625667572021, -0.641613781452179, -0.24539509415626526, -1.1594171524047852, 2.024722099304199, 1.4229263067245483, -1.2096515893936157, 1.18757164478302, -0.4213385283946991, -0.09276046603918076, -0.3906360864639282, 0.32764938473701477, 0.5627915859222412, -0.7354503870010376, 1.0429892539978027, -0.2516310513019562, 0.4061100482940674, 1.0937579870224, 1.9668548107147217, 0.6220248937606812, -0.4034467041492462, -0.12901437282562256, -0.087590791285038, 0.9531611800193787, -0.02341020293533802, 0.8127734065055847, 0.5308603644371033, 0.39214152097702026, 0.8768171072006226, -0.1179799810051918, -0.6958360075950623, -0.08782492578029633, 0.991192102432251, 1.05316162109375, 1.5033084154129028, -0.8733856678009033, -1.2814226150512695, 0.051408059895038605, 1.146470069885254, -0.07229632884263992, 0.8348703384399414, 0.42143380641937256, -0.2610929310321808, 0.9840702414512634, -0.48412126302719116, 0.5301610827445984, -0.24602337181568146, -0.5629846453666687, -0.9011794328689575, 0.7663253545761108, 0.285744309425354, -1.3896540403366089, 0.813595175743103, 0.5398550033569336, -1.4419680833816528, 0.2683210074901581, 0.08273708820343018, -1.5031929016113281, -1.9937880039215088, -0.5480549335479736, 0.8144537806510925, -0.530667245388031, -0.7549601197242737, 1.6137462854385376, 0.8061066269874573, 3.1218132972717285, -1.7967697381973267, 0.7586671113967896, 0.29097041487693787, -0.581048309803009, 0.4167138934135437, 0.4290783405303955, -1.160550594329834, 1.4881200790405273, -0.3583008050918579, 0.8391028642654419, -0.39563775062561035, 1.2523990869522095, -0.350832998752594, -1.128920078277588, 1.0651965141296387, -1.6551291942596436, 0.6373022198677063, -0.8371967077255249, -0.2545940577983856, -0.07490552216768265, -0.5544443726539612, 0.3938034474849701, 1.4644033908843994, 0.6627893447875977, -0.9653831720352173, -0.10260355472564697, -0.7984984517097473, -2.161821126937866, -1.755712866783142, -0.7055662274360657, -1.2166212797164917, 0.6726545095443726, 0.764571487903595, 0.31289035081863403, -1.4657021760940552, -0.0971238762140274, 0.0002556077961344272, -0.5667784214019775, -1.4147253036499023, -0.06757300347089767, 1.1331219673156738, -0.4447691738605499, 0.7411748766899109, -0.7409497499465942, 0.7050660252571106, 1.4093067646026611, -0.3435841202735901, -1.7699211835861206, -1.7922557592391968, -1.0752360820770264, -0.736550509929657, -2.775763988494873, -1.0402827262878418, 0.6430092453956604, -0.213924840092659, -0.8959898948669434, -2.061768054962158, 0.23493459820747375, -1.395169734954834, -0.17476311326026917, 0.34423208236694336, 0.494499146938324, 0.12198653072118759, 1.3564279079437256, 0.7676032781600952, 1.0170106887817383,0.2759431302547455, 0.22733838856220245, 0.8085190057754517, 0.06586667895317078, -1.0810660123825073, 1.3978875875473022, 0.0676366463303566, -0.5710737109184265, 2.3214101791381836, -0.31675711274147034, 0.8402789235115051, 1.3929555416107178, -0.7704640030860901, -0.30536848306655884, 0.7163468599319458, 0.1597607284784317, -1.3581397533416748, -0.5458253026008606, -1.2323791980743408, 0.354978084564209, -2.0267493724823, 0.16130514442920685, -0.3372887074947357, -0.3226206302642822, -1.0339089632034302, -1.3146780729293823, -0.06869571655988693, 1.1066420078277588, -0.22585107386112213, 0.9199158549308777, -0.3292832672595978, -0.8790549039840698, -0.6610155701637268, 0.14308813214302063, 0.08865055441856384, -0.7894478440284729, 0.06781133264303207, -1.3603487014770508, -0.1584135890007019, -0.41870957612991333, 0.5215525031089783, 0.7596070766448975, -0.3239614963531494, 0.24650253355503082, -0.5681624412536621, -0.443688303232193, 0.6050768494606018, -0.00348252197727561, 2.0301506519317627, -1.3617758750915527, 0.3930289149284363, 1.1224445104599, 0.19795791804790497, -0.2994332015514374, 1.201195240020752, -0.6250950694084167, -1.0219329595565796, -0.012784653343260288, 0.169631227850914, -1.0370895862579346, -0.794836163520813, -0.10800512880086899, 1.7157025337219238, -0.37885168194770813, -0.5801583528518677, -0.10595729202032089, -0.2595713138580322, -0.4330618977546692, -1.400607943534851, -0.5887740254402161, -1.3022078275680542, 0.0656617283821106, 0.19446049630641937, 0.7109612226486206, -1.1257778406143188, -1.70765221118927, 1.5033215284347534, 0.23377643525600433, -2.000135898590088, 1.1745330095291138, -1.1041796207427979, 0.2874129116535187, -1.317428708076477, -0.19078540802001953, -0.21040675044059753, -0.9150257706642151, 0.27452707290649414, -0.0055251289159059525, -1.7189172506332397, -0.5017772912979126, 0.9677766561508179, 0.4922860264778137, 0.43207961320877075, 0.45276421308517456, -0.26017674803733826, 2.944723606109619, -0.25813499093055725, 1.4113569259643555, -1.5482828617095947, -0.6165332794189453, -0.4437899887561798, -0.7817089557647705, -0.2200002521276474, 0.4552081525325775, -0.622263491153717, 0.3829461932182312, -1.0469073057174683, 1.155548095703125, -1.1233197450637817, 0.015525500290095806, 1.22166109085083, -0.10545527935028076, -2.1521787643432617, -0.374954491853714, -0.8675704002380371, 0.36605024337768555, 0.1837126761674881, 0.14570318162441254, -0.4171552062034607, 0.07691596448421478, 1.3370256423950195, -0.30584731698036194, -0.30491307377815247, -1.057435393333435, -1.3079164028167725, 0.33825749158859253, 0.6129353046417236, -0.020468611270189285, -0.9308547377586365, 1.2005540132522583, 0.3236226737499237, 1.052394151687622, 1.0278222560882568, 0.7108228206634521, 0.6722753643989563, 0.7974400520324707, 2.663214683532715, -0.02112465910613537, 2.4405298233032227, -0.205564484000206, -0.3837776780128479, 0.8452321887016296, 0.5255430936813354, -1.354914903640747, 0.13891597092151642, 0.46605879068374634, -0.5343753695487976, -0.6830325126647949, 0.040955763310194016, 0.43499118089675903, -0.25481241941452026, 1.0568958520889282, -0.015591039322316647, -2.4479820728302, -0.07496581971645355, 0.7054945826530457, 0.8391988277435303, 0.32093948125839233, -0.14575161039829254, 0.7059656977653503, 0.3008079528808594, -0.07684821635484695, -1.5679879188537598, 0.8423724174499512, -0.33305129408836365, -1.5048532485961914, 0.7039641737937927, -0.11697425693273544, 0.030102187767624855, -1.2884149551391602, 1.0394426584243774, -0.42216795682907104, 0.4744715392589569, 0.8790180087089539, -0.04358021542429924, -0.24201586842536926, -0.4630572199821472, 0.15962691605091095, 0.7591912150382996, -0.7444291710853577, 0.7596778869628906, -0.22694167494773865, -0.5222982168197632, 0.25046005845069885, 0.8937973380088806, -0.22825664281845093, -1.80848228931427, 1.3860212564468384, 0.6869720220565796, -1.2371519804000854, 0.6895752549171448, 1.5102064609527588, -0.6206377744674683, -0.7499287724494934, -1.1584084033966064, -0.016004318371415138, 0.05717207491397858, 0.07868336141109467, 0.7437389492988586, -0.5613594055175781, 0.27965036034584045, 1.060808539390564, 0.28247424960136414, -0.691230297088623, 2.2329909801483154, -0.3413757085800171, 0.45017221570014954, 0.5863032937049866, 0.9948636889457703, 1.346261978149414, 1.2605761289596558, -1.3028371334075928, 0.7706832885742188, -0.4684984087944031, 0.7539286613464355, -0.9236618280410767, 0.6162471771240234, 0.050153832882642746, -0.29591086506843567, 1.3616187572479248, -0.4735174775123596, -1.8475925922393799, -0.04024212062358856, 1.893950343132019, -0.43023842573165894, 2.0077712535858154, 0.7685759663581848, -0.12809036672115326, 1.399756908416748, -0.07466902583837509, 0.8052807450294495, -0.8498088121414185, 0.8828974366188049, -0.38970768451690674, 1.4178686141967773, 1.5349246263504028, -0.24809516966342926, 0.026474395766854286, -0.43083176016807556, -0.22659824788570404, -1.930429458618164, -1.0382320880889893, -0.42418867349624634, -1.1959950923919678, -0.22889813780784607, 0.4338840842247009, -0.3317365050315857, -0.252463698387146, 0.20222370326519012, -1.2377461194992065, 0.6726632118225098, -1.598530650138855, -2.237480401992798, 2.2723233699798584, 0.1589750051498413, -0.7851131558418274, -0.6269485354423523, 0.01597079448401928, -0.7177801728248596, -1.5291666984558105, -0.33408382534980774, 0.6213129758834839, -0.27196434140205383, -0.9577853679656982, -0.17163771390914917, -0.03999944403767586, -1.1686710119247437, -0.2025771290063858, 0.3647274672985077, -0.0587940588593483, 0.0291097704321146, -0.9005841612815857, -1.4717698097229004, -0.08551472425460815, -0.33969101309776306, -1.5115522146224976, 0.421186238527298, -0.7002263069152832, 0.24399541318416595, -1.039589524269104, 0.9762930274009705, -0.42144209146499634, 0.6447209119796753, -0.8031951785087585, -1.4969638586044312, -1.1694037914276123, 0.44726482033729553, -2.1063392162323, -1.2299257516860962, 0.19059336185455322, -0.19982536137104034, 1.221371054649353, -1.4987434148788452, 0.954046368598938, 1.3617994785308838, 1.9749343395233154, 1.044108271598816, -0.45073792338371277, -1.0088075399398804, -1.5151528120040894, 0.1491449922323227, -0.15880227088928223, -1.1301676034927368, 0.19266065955162048, 1.283307433128357, 0.5607054829597473, -1.132053256034851, 0.9801673889160156, -1.7369447946548462, 2.522023916244507, 0.11381684988737106, -1.1431622505187988, -0.3079226613044739, -0.02560385689139366, -0.7477001547813416, -0.35642194747924805, 0.9299072027206421, 0.41082578897476196, 1.2577482461929321, 0.21280847489833832, 0.21515539288520813, 0.001410643570125103, 1.2832896709442139, 0.9118039608001709, 0.45135772228240967, -0.17647437751293182, 0.10003817081451416, 1.2641243934631348, 0.4739455580711365, -0.09329804033041, 0.13256296515464783, 0.059780366718769073, -1.675307273864746, 1.7277299165725708, -0.5968847274780273, 1.8818163871765137, 0.9658488631248474, 1.6620100736618042, 1.2323874235153198, 1.0742744207382202, -0.057652633637189865, -1.1752411127090454, 2.0446503162384033, 0.8739678263664246, 1.0076297521591187, 0.7780562043190002, -0.9623212218284607, -0.04193798452615738, -0.406357079744339, -1.7856258153915405, 0.5729252696037292, -0.8894615769386292, 1.4139302968978882, -0.4045787751674652, -1.4192341566085815, -0.7681483626365662, -1.3613871335983276, -2.6325459480285645, 0.1777186095714569, 1.2982040643692017, -1.8974894285202026, 1.150920033454895, 0.03501071035861969, 0.05434811860322952, -0.9249972701072693, -0.12226668745279312, -0.30022457242012024, 0.5222684144973755, 1.475637674331665, 0.19754283130168915, -1.0709362030029297, -0.24381592869758606, 1.9935864210128784, 0.32553157210350037, 0.5969431400299072, 0.036618057638406754, 0.6924362182617188, -0.1326826959848404, -0.021698489785194397, 0.777678906917572, 1.203311800956726, -0.27010342478752136, -1.4111000299453735, -0.9341485500335693, -2.5191233158111572, -0.014349877834320068, -0.3457161784172058, 0.46627482771873474, 1.204741358757019, -0.9680691361427307, -0.13983985781669617, 0.29761946201324463, -0.11613212525844574, -0.17423322796821594, 0.8485185503959656, -0.8048476576805115, -1.2765164375305176, 1.85976243019104, -0.27748990058898926, -2.0512969493865967, -0.934514045715332, 0.14629188179969788, -0.14867594838142395, -0.5327825546264648, 0.020204618573188782, 0.4906969368457794, -0.28001853823661804, -1.0615575313568115, -0.7620670795440674, -0.32496288418769836, 0.15228083729743958, 0.5059142112731934, 1.469596266746521, 0.2659119963645935, -1.4908843040466309, -1.004393458366394, 0.3240762948989868, 0.9638779163360596, -0.08983337134122849, -0.4134518504142761, 0.5105448961257935, 0.9298027157783508, 1.4331068992614746, -0.1489855945110321, -0.3646010160446167, 0.009838655591011047, -1.5163770914077759, 0.4892819821834564, 0.582725465297699, 0.26357877254486084, -1.1720510721206665, 0.4504058361053467, 1.5105688571929932, 0.17434966564178467, 0.18500161170959473, 2.27270245552063, -0.5114508271217346, 0.13420869410037994, 0.5124810338020325, -1.0657341480255127, 0.40332239866256714, -0.8427419066429138, 1.4399393796920776, -0.24957333505153656, 1.0421007871627808, -0.1093556135892868, 1.0491535663604736, -0.4130123555660248, 1.2392325401306152, -0.5230810046195984, 1.99565851688385, 2.0016095638275146, 0.5357887148857117, -1.0852676630020142, -0.10117699205875397, -0.20757608115673065, 0.6644294261932373, 1.152113676071167, -0.11689209938049316, 0.6258598566055298, -0.4913567900657654, -0.9442599415779114, -1.0982826948165894, -0.227497860789299, 1.0586693286895752, 0.32198449969291687, -0.1728964000940323, -0.7015251517295837, 0.9173167943954468, 0.5622948408126831, -0.7231076955795288, 0.28802433609962463, -0.42161431908607483, -0.1552494615316391, -1.2256946563720703, -0.00472315214574337, -0.9864890575408936, -0.24393974244594574, -2.6695239543914795, -0.6634103655815125, 0.7526135444641113, 0.5096832513809204, 0.08407985419034958, 0.0573432631790638, -0.013678156770765781, -1.0212634801864624, 0.6849406361579895, -0.9775760173797607, 0.05029475316405296, -1.5996410846710205, 0.728835940361023, -0.9325684905052185, -3.7970168590545654, -1.846966028213501, 1.2054940462112427, -1.2549856901168823, -0.9391381740570068, -0.9963999390602112, 0.40594902634620667, 0.9186541438102722, 0.9358623623847961, -1.0129905939102173, 1.1290814876556396, 1.3368192911148071, -0.9695120453834534, -0.5027217268943787, -0.6207431554794312, 1.0784215927124023, 0.12517458200454712, 2.0262651443481445, 1.7260921001434326, 1.6558938026428223, -0.13750673830509186, 0.08315733075141907, 1.1841344833374023, -0.9472866058349609, -0.9762137532234192, -2.55810809135437, -0.22479695081710815, 0.7724712491035461, 0.6405773758888245, -0.7069220542907715, 0.722794234752655, 2.0284383296966553, -0.6952884793281555, -0.3428124189376831, 0.856074869632721, -0.27228090167045593, 0.06298405677080154, -0.06455832719802856, 0.1721697747707367, -0.40316855907440186, -1.0886337757110596, -1.363294243812561, 0.4130248427391052, -0.7744693160057068, -0.7297961115837097, 1.160874605178833, -0.41687777638435364, 0.5012897849082947, 0.37825843691825867, -0.39632394909858704, 0.08698981255292892, -0.30206364393234253, 1.2822215557098389, 0.04488609731197357, -0.3879280984401703, 1.2463434934616089, 0.6901986598968506, -0.09165635704994202, 1.2805171012878418, -1.6823041439056396, -0.44262635707855225, -0.9507811069488525, -1.4429336786270142, -0.051316998898983, -0.38084858655929565, 1.519568920135498, 0.04240787401795387, -1.3709709644317627, -0.9961370825767517, -0.9133378863334656, 0.6133418679237366, 1.0894757509231567, 0.9103439450263977, -0.7014421820640564, 0.13837885856628418, -0.1675906628370285, 0.4645448625087738, 0.556890606880188, 0.051723286509513855, -1.9380416870117188, -0.7669622898101807, -1.8666880130767822, 1.0815067291259766, 1.1760860681533813, 0.012840385548770428, -0.4789154529571533, 0.671730101108551, -1.5707268714904785, -0.09156513214111328, 0.47525569796562195, 0.756725549697876, -0.30547896027565, 0.08147407323122025, -1.5565143823623657, -1.0718938112258911, -0.43623584508895874, -1.633470892906189, 0.2781197726726532, -0.9348036050796509, -0.13428640365600586, 0.38985151052474976, 0.05663495883345604, 0.07877818495035172, -0.6122056245803833, 0.22611653804779053, -1.1845529079437256, -1.1092251539230347, -1.190717101097107, 0.45246317982673645, -0.9358522891998291, 0.8919274210929871, 0.18282374739646912, -0.4918440580368042, 1.1324626207351685, -0.6443055272102356, -0.4544437825679779, -0.09183770418167114, -0.6177263855934143, -1.096350908279419, -1.2881215810775757, -0.038739703595638275, -0.051358211785554886, 2.95823335647583, 1.2290749549865723, -0.9379467368125916, 0.86658775806427, -1.7001087665557861, -3.0471155643463135, -0.7226195335388184, -0.946428120136261, -0.5026091933250427, -0.3002597391605377, -0.8329758644104004, -1.0126128196716309, -1.5748732089996338, -0.14179398119449615, -0.9042802453041077, 1.8664172887802124, 1.824265718460083, -3.706285238265991, -0.812869668006897, 0.32398104667663574, -0.7028282880783081, 0.6239628791809082, -1.5643415451049805, 0.021328939124941826, 1.4801642894744873, 0.6541654467582703, 0.3994036316871643, -1.8519415855407715, 1.3120352029800415, 0.1646733582019806, -1.0693459510803223, -0.3050614893436432, -0.19283373653888702, 1.2031522989273071, -1.2342100143432617, -1.1485565900802612, -0.6954340934753418, -0.5550722479820251, -1.0055034160614014, -0.21186839044094086, 0.6194276809692383, 0.8238816857337952, 0.12562720477581024, 0.06739916652441025, 2.0787627696990967, -0.33865347504615784, -0.717936635017395, 0.06893109530210495, -0.7409762144088745, -1.066051959991455, 0.3781960904598236, 0.2640245854854584, -0.04877258464694023, 2.2136082649230957, 0.3980749547481537, -0.6240140199661255, -1.2020788192749023, -0.49883562326431274, -1.3568050861358643, 0.9256665110588074, -0.803889811038971, 0.9905645847320557, -0.8854333162307739, -0.45431703329086304, 0.2971067726612091, -0.6928515434265137, -1.9616127014160156, 0.2913694977760315, -0.4134928286075592, -1.002907156944275, -1.2468165159225464, -0.05442887544631958, -0.8036088943481445, -1.164482593536377, -3.058319568634033, -0.2737196087837219, -0.1633516550064087, 1.126125454902649, 0.026944248005747795, -0.5808191895484924, 0.035995062440633774, -0.48715078830718994, -0.8068931698799133, -1.0322141647338867, -1.237187147140503, 1.0493407249450684, -0.5701891183853149, -0.2519337832927704, -0.6363653540611267, 1.3473411798477173, 0.5468118190765381, -1.2925825119018555, 1.1896381378173828, 0.23162981867790222, -0.8208529949188232, 0.14567221701145172, -0.7768197059631348, -0.7868624329566956, 0.6907272934913635, -0.9960556030273438, 0.8023239970207214, -0.0290460754185915, 0.5633772015571594, 0.07399056106805801, 0.1288917064666748, 1.7031906843185425, 2.662721633911133, -0.07347472757101059, 1.315494418144226, -0.31208255887031555, 0.5601041913032532, 0.5546759366989136, 1.795273780822754, -0.501926600933075, 0.3928082585334778, -1.4137431383132935, -1.6088029146194458, 1.4604170322418213, -0.3867703378200531, -1.6671746969223022, 1.1459449529647827, -2.423792600631714, -0.7000697255134583, -0.030625367537140846, -0.6631996631622314, 0.23222891986370087, 0.39063411951065063, 0.604027509689331, -1.3413188457489014, 0.49349454045295715, -1.4467161893844604, 0.02272111363708973, -0.7627412676811218, -1.3154174089431763, -1.3534685373306274, 1.080155611038208, -0.5921106934547424, -0.06778863817453384,1.6084275245666504, 0.9460328221321106, 1.0908126831054688, -1.269880771636963, 0.6677347421646118, -1.3186383247375488, -0.6099237203598022, -1.9246032238006592, -0.1407081037759781, 0.8975345492362976, 0.2750260829925537, -0.2042740434408188, -2.301138401031494, -1.2356090545654297, 0.37821248173713684, 0.5403173565864563, 0.872315526008606, -0.8511752486228943, 1.3493598699569702, -0.5566791296005249, -0.41151753067970276, -0.625844419002533, 0.6830285787582397, -0.9548120498657227, 0.47729650139808655, 1.2793091535568237, 0.4406825006008148, 1.0988712310791016, 0.6755485534667969, 1.014804482460022, 0.9591502547264099, 0.1802215576171875, 0.42190736532211304, -0.7966063618659973, -0.029975805431604385, -0.4892694652080536, -0.21010024845600128, -0.04777872934937477, 0.6602339148521423, -1.4873263835906982, 0.0017058609519153833, 0.12510238587856293, 0.8977320790290833, 0.7751132249832153, -0.07183884084224701, -0.5369117856025696, 0.533854067325592, -0.5583081245422363, -0.28552955389022827, -0.27237260341644287, 0.48447734117507935, -1.3580296039581299, 0.6174036264419556, 0.06594301760196686, 0.36476460099220276, -2.101731061935425, 1.8329941034317017, -0.9468764662742615, -0.22405894100666046, -1.5427742004394531, 1.9570220708847046, -0.2749195694923401, 0.6078005433082581, -1.4204121828079224, -0.5127604007720947, -1.10904860496521, -0.6098837852478027, 0.5138894319534302, 1.4510209560394287, -0.06193571165204048, 0.4082811772823334, -2.736572504043579, -0.059652626514434814, -0.8445574641227722, -0.3709193170070648, -0.9305031299591064, -1.2282330989837646, 0.3505038917064667, 0.5408838987350464, 0.2040925920009613, -1.174362063407898, -0.22934405505657196, -0.7409008145332336, 1.1326183080673218, -1.4805607795715332, -2.3223695755004883, 0.6001764535903931, 0.9770926833152771, 0.8561793565750122, 0.15442852675914764, -0.2230379283428192, -0.4607117176055908, 0.2684352993965149, -0.4663431644439697, -0.8680515289306641, -1.7358535528182983, 0.5979013442993164, 0.8781825304031372, 0.9078559875488281, -1.8507606983184814, -0.2604331970214844, 0.3626272678375244, -1.2735379934310913, 0.055010560899972916, -0.04144098609685898, 0.8982192277908325, -1.3220126628875732, 0.7709842324256897, 1.0065373182296753, 0.6670200228691101, 0.6594529747962952, 1.1488782167434692, -0.3218682110309601, 0.5331389904022217, -0.9747026562690735, 0.05544540286064148, -0.6229236721992493, -1.2202749252319336, 0.34896689653396606, 0.8188266158103943, -1.0781433582305908, 1.4943859577178955, -0.9097751975059509, 1.6188483238220215, 0.5527724623680115, 1.600177526473999, -0.4862789809703827, -0.7006863355636597, 1.0539751052856445, -0.7449480295181274, -0.1946820467710495, -1.6017669439315796, 0.9206735491752625, 0.504697322845459, -0.8731178045272827, 0.8897388577461243, 0.17679139971733093, -0.7914062738418579, -0.7883844971656799, -0.9177232384681702, 0.514638364315033, -0.11917448043823242, 1.006001353263855, -0.26238930225372314, 0.34822309017181396, -0.6053542494773865, -0.0473029688000679, 0.21546989679336548, 1.0227056741714478, -0.42437800765037537, -0.5430347919464111, 1.577705979347229, 0.6282734870910645, 1.3722546100616455, -0.1976596564054489, 0.8418416976928711, 0.021006489172577858, 0.5612190365791321, 0.2092261165380478, -0.35027655959129333, -0.302054762840271, 1.0560835599899292, -0.8992435932159424, 0.6699095964431763, 0.7626084089279175, 1.0316756963729858, -1.050696849822998, 0.9288585186004639, -0.03721686080098152, 0.6351494193077087, -0.6125960946083069, 0.7980482578277588, -0.18006622791290283, 0.6521767377853394, -0.22456400096416473, 0.9734818935394287, 0.19433490931987762, 0.835617184638977, -1.015360713005066, 1.5580661296844482, -0.2347625344991684, -0.09803981333971024, -1.6373611688613892, 0.25385305285453796, 0.10515131056308746, 0.7663154602050781, 0.7901588082313538, 2.343249559402466, -0.5719259977340698, -0.7781572937965393, -0.7509758472442627, 1.6086970567703247, 0.27375170588493347, -1.145675539970398, 0.41438090801239014, -0.08058322221040726, 1.178454041481018, -0.45107218623161316, -1.1352051496505737, 0.3824276328086853, 1.132500410079956, -0.006934213452041149, -0.2854752838611603, 0.7127864360809326, 0.7912495136260986, -2.6896865367889404, -0.3251606523990631, 0.18404367566108704, 0.627601683139801, 0.7209868431091309, 0.5184066891670227, 0.6944921016693115, -0.5518170595169067, 0.6849256753921509, 0.18886855244636536, -0.18796131014823914, 1.0695675611495972, -0.9194974899291992, -0.2719438374042511, -0.9627880454063416, -0.3848208785057068, -0.5525501370429993, -0.45218244194984436, -1.014289379119873, 0.33250564336776733, 0.06349172443151474, -0.502055287361145, -0.062353383749723434, 2.177851438522339, -1.1249738931655884, 0.9984826445579529, 0.6100653409957886, -0.46446624398231506, -0.14318028092384338, -0.10686942934989929, 0.7280798554420471, -0.22633075714111328, -0.9837703108787537, 0.6050372123718262, 0.27324116230010986, -0.04385041072964668, -1.1225985288619995, 1.5210390090942383, -1.1889116764068604, -0.00734232272952795, 1.0370228290557861, 0.13974110782146454, 2.1854991912841797, -0.5764122605323792, 1.0446088314056396, 0.16614653170108795, 0.43826955556869507, -0.577720582485199, -1.0930534601211548, -0.4209333062171936, -0.32822638750076294, 0.8052898049354553, 0.9495745897293091, -0.22140321135520935, 1.536562204360962, -0.4766591191291809, -1.7389707565307617, -0.07204469293355942, 0.5080832242965698, 1.4451874494552612, 0.32077762484550476, 0.8423900008201599, -0.3263871967792511, 0.391702264547348, 0.1371258795261383, -0.4068228304386139, 1.7035694122314453, -1.0882642269134521, 0.8754494786262512, -0.024457154795527458, -1.0658600330352783, -1.4400794506072998, 0.30305808782577515, 0.8944989442825317, 0.7112008333206177, -1.6300077438354492, -0.9041943550109863, -1.8243244886398315, 0.7455897927284241, -1.0189632177352905, -1.609115481376648, 0.13888408243656158, -0.6087268590927124, -0.7933457493782043, -0.8609066009521484, 1.315445899963379, -0.040957286953926086, -0.5407764911651611, 0.21511968970298767, -0.8952437043190002, -0.12016328424215317, 1.4888896942138672, -0.6353124380111694, -0.2833486497402191, 0.6220099329948425, 0.9757431745529175, -0.9865791201591492, 1.8836301565170288, -1.1641210317611694, 1.4571517705917358, -1.0207712650299072, 1.197340965270996, -1.2104551792144775, 1.0054367780685425, 0.9979689717292786, 0.3705267608165741, 0.060966331511735916, -1.332219123840332, -0.8829978108406067, -0.5061601400375366, 2.337172746658325, -0.06857552379369736, 0.2526608109474182, 1.0036948919296265, -0.21303512156009674, -0.44562989473342896, 0.21667294204235077, 0.46637117862701416, 0.6161444187164307, -0.6841418743133545, 0.22695788741111755, -2.480497360229492, 0.9099594354629517, -0.10208448767662048, 0.8274369239807129, 0.363984614610672, 2.8211302757263184, 0.6174241900444031, -0.2867540717124939, -1.145920991897583, 0.6986541748046875, 0.34877288341522217, 0.9601255059242249, -1.4895721673965454, -1.7155826091766357, 0.12531472742557526, -0.017052682116627693, 0.9923937320709229, -0.42303910851478577, -0.831082284450531, 0.6534690260887146, 0.3220108449459076, 0.8592017292976379, 1.7652760744094849, 0.31227603554725647, -0.8129943609237671, 0.2254871279001236, 0.6766112446784973, -1.5478066205978394, -0.7923154830932617, 0.2342817485332489, -1.1467700004577637, 0.40981265902519226, -1.0758299827575684, -1.0612083673477173, -1.9065090417861938, -0.503415048122406, -0.3633720874786377, 1.1541084051132202, -1.9135277271270752, -1.4975438117980957, 0.27217522263526917, 0.4585931599140167, -0.3230949640274048, 0.17687012255191803, 0.11914658546447754, -0.17411112785339355, 0.646613597869873, -2.446972608566284, 1.7849054336547852, -1.7285139560699463, -1.4685636758804321, -0.4473296105861664, -0.4164807200431824, 0.4420527517795563, -0.043058037757873535, 2.088094711303711, 2.0430104732513428, -1.6084988117218018, 0.5031100511550903, 0.3332343101501465, 1.093156337738037, 0.6620869636535645, -2.527660846710205, 0.12872058153152466, 0.8669702410697937, -1.0569874048233032, 0.5532603859901428, 0.6979382634162903, -0.9543694853782654, 0.07212191820144653, -0.7435635328292847, 1.4033735990524292, -0.8374363780021667, -0.7969957590103149, 2.0782573223114014, -0.4779047966003418, -0.46648383140563965, 0.9255260825157166, -0.524514377117157, 0.2767593264579773, 0.7275776267051697, 1.1499699354171753, -0.5362938642501831, -2.1202950477600098, 1.0537335872650146, 0.5042433738708496, -0.5513375997543335, -0.6684633493423462, 0.04519916698336601, 0.42469239234924316, 0.7379332184791565, -0.316583514213562, 1.5730621814727783, 0.07526570558547974, 0.10408965498209, 1.774605631828308, -0.7965874671936035, -2.067018985748291, 0.594656765460968, 1.4551900625228882, 0.5145246982574463, -0.7543981075286865, 1.3586801290512085, 0.22427847981452942, 0.04401369020342827, 0.3565187156200409, -0.769607663154602, -0.9360424280166626, -0.2082897126674652, -0.9548380970954895, -0.2090919017791748, 1.9032225608825684, -0.7354087829589844, -1.8955146074295044, 0.38482606410980225, 0.008789419196546078, 0.3404351472854614, 1.4987678527832031, 1.9403475522994995, 0.4439668655395508, 1.5893583297729492, -0.6321508884429932, 0.04087845981121063, 0.07673671096563339, -0.132492795586586, 1.3809324502944946, -2.260242223739624, 0.8152600526809692, 0.7748892903327942, -0.04208943247795105, -0.1376657783985138, 0.14951889216899872, 0.9139845371246338, 1.9888834953308105, -0.8125941157341003, 0.7949867844581604, -0.9433197975158691, -1.1286565065383911, -0.506039559841156, -0.2703445255756378, 0.05661916732788086, -1.393202781677246, 0.1752283126115799, -1.133665919303894, -0.35541000962257385, -1.4378067255020142, 0.09646730124950409, 0.549950361251831, 1.2157145738601685, 1.1221390962600708, -0.7638704180717468, 0.8077789545059204, 0.5576744079589844, -0.27321478724479675, 0.8099126219749451, -2.5899875164031982, 0.10101302713155746, 0.18246455490589142, -0.13919752836227417, -0.060193441808223724, 0.013675876893103123, 1.2373610734939575, -1.277309536933899, -1.505039930343628, -2.705848217010498, 1.1098542213439941, 0.6965665817260742, 0.5827547907829285, -0.820183515548706, -0.8826172947883606, 0.6978957653045654, 1.6358091831207275, 0.005438170395791531, 0.9719974398612976, 2.6504409313201904, -0.36470454931259155, 0.16235889494419098, 1.8278701305389404, 0.9800227880477905, 0.21562066674232483, 0.6585524082183838, 0.960176408290863, 1.2827181816101074, 1.146032452583313, 0.07973092049360275, 0.6863148808479309, -0.007537241093814373, -0.581308901309967, -0.5597913861274719, 1.6126494407653809, 0.38784754276275635, -0.9748775362968445, -0.6968530416488647, 1.104840874671936, 0.5419602394104004, -1.1744943857192993, 0.44379040598869324, -0.5722488164901733, -0.09470131248235703, 0.8909420967102051, 0.10610615462064743, 0.650132954120636, -0.4439186751842499, 0.03249204903841019, 2.3510308265686035, 0.5108466148376465, 0.8143020868301392, 0.9068869948387146, -0.52064448595047, -1.4799344539642334, 0.5569291710853577, 1.25213623046875, 0.22235150635242462, -0.9282664656639099, 0.5843960642814636, 1.0096526145935059, -0.41383039951324463, -1.6792877912521362, -0.5050851702690125, 1.4650007486343384, -0.9776894450187683, -0.6822090148925781, -0.7663251161575317, -1.7490878105163574, 0.3188040256500244, 0.2812509536743164, 0.10266371071338654, 0.1875283122062683, -0.7663653492927551, -1.143490195274353, 0.3345777094364166, -0.11012513935565948, -0.35172566771507263, -1.1426527500152588, -0.40426066517829895, 0.2911964952945709, -0.3864605724811554, -0.2632461488246918, 0.05154870077967644, -0.44450119137763977, -1.621302604675293, 0.9656539559364319, -0.6148766279220581, -0.4422946870326996, 1.791125774383545, 0.6872227191925049, -0.1959138959646225, -0.970851480960846, 1.4420257806777954, -0.9594331979751587, 1.1277921199798584, -0.3829268217086792, 0.5900583267211914, 0.09205328673124313, -0.13368432223796844, -0.3333124816417694, -1.3050400018692017, -0.7698356509208679, 0.42067751288414, 0.2835474908351898, 0.775964617729187, -1.3824290037155151, -0.5626915693283081, 0.11738666146993637, -0.21090443432331085, 0.28001296520233154, -0.4146668016910553, -2.1412699222564697, -0.43131759762763977, 0.8893797397613525, -0.7439267039299011, -0.37895339727401733, 0.533160388469696, 0.9230329990386963, -1.3910588026046753, 0.3867364525794983, -0.0816870927810669, 0.3061969578266144, 1.3318501710891724, 0.48234307765960693, -0.8763262629508972, -0.6764206886291504, 0.8622604012489319, 0.7564119100570679, 0.5404233336448669, 1.479318380355835, 0.3201580345630646, -0.4836347699165344, 0.4744700789451599, 0.5927971005439758, -0.99949049949646, 0.7024921774864197, -0.010735027492046356, -0.8412494659423828, 0.04259481653571129, -0.2850961983203888, -0.9706072211265564, 0.47977715730667114, -0.5743458271026611, -0.21985645592212677, -0.9108624458312988, 1.3178911209106445, 0.07040096819400787, -0.05676835775375366, 0.46159127354621887, 0.4703606963157654, 1.5580816268920898, 0.6987341642379761, 0.9532100558280945, -1.1860729455947876, 1.065573811531067, -0.6648626327514648, 0.6030547022819519, 1.973741888999939, -0.09826762229204178, -0.05814157426357269, 0.4331076443195343, 0.8468711972236633, -1.047662377357483, -0.8275336623191833, -0.08599145710468292, 0.22419777512550354, 0.22669780254364014, -0.31352120637893677, 0.3455999791622162, 0.11023841798305511, -0.3398609161376953, 0.0945354625582695, -0.5854671001434326, 0.640220046043396, 0.022231265902519226, -0.23790428042411804, -0.5506563782691956, 1.4304273128509521, 1.2297295331954956, 0.47987860441207886, 0.6092092990875244, -0.8162793517112732, 0.7545724511146545, -0.12891864776611328, -0.4087192118167877, -0.6845815181732178, 0.2850717008113861, 0.7640952467918396, 1.14008629322052, -0.2336016297340393, -0.8930046558380127, -0.6739396452903748, 1.830496072769165, -1.1774988174438477, -0.22418414056301117, 1.1350547075271606, -0.4167617857456207, -1.1148887872695923, 2.2971858978271484, 0.06434177607297897, -0.7233536243438721, -0.44746091961860657, 1.180661916732788, -1.3618911504745483, -1.3453177213668823, -0.8076579570770264, -0.9162730574607849, -0.13132452964782715, 2.0653560161590576, -0.5722118020057678, -0.7555540204048157, 0.5003328919410706, 0.1754697859287262, 1.0768499374389648, -0.08945679664611816, 1.3659690618515015, -1.2080533504486084, 0.6184517741203308, -0.15558171272277832, -0.2913418412208557, -0.6473309397697449, 0.07697276771068573, -2.5795676708221436,1.5442131757736206, -0.582919716835022, 0.49576589465141296, 1.0914579629898071, 0.205585777759552, 0.44446924328804016, -1.2725422382354736, -0.874988853931427, 1.0853474140167236, 0.6201907396316528, -0.5369606614112854, 0.34373900294303894, -0.7939163446426392, 0.039030127227306366, 0.7023033499717712, 1.3785223960876465, 1.2905842065811157, -1.3000930547714233, 1.4208430051803589, 1.329713225364685, 0.016362667083740234, 0.6801381707191467, 0.8409186601638794, -0.9359179139137268, -0.6289897561073303, -0.3299519419670105, -1.141353726387024, 0.663020133972168, 0.38736778497695923, 0.9776961803436279, -0.5153890252113342, 2.6922271251678467, 0.4937732517719269, -0.17910900712013245, 1.3677096366882324, -0.6795288324356079, -0.7589612007141113]; - let ground_truth = vec![2.153256416320801, -8.757116317749023, 21.87359046936035, -0.07722868025302887, 2.5499470233917236, 4.972344398498535, 3.497126817703247, -2.2173497676849365, -3.1898372173309326]; + let input = vec![-1.1441841125488281, -0.3753563463687897, 0.7898056507110596, -1.393734097480774, -0.3568798899650574, 0.9762657284736633, -1.5152218341827393, 0.7137635946273804, 0.9716352224349976, -1.1083635091781616, -0.833057701587677, 0.47789332270622253, 0.5295599699020386, 0.8025445342063904, -1.43166983127594, -0.5103325247764587, -0.2988021671772003, 0.3167966306209564, -1.5173636674880981, -0.4598764479160309, 0.22498439252376556, -0.26091915369033813, -1.1264656782150269, -0.32812589406967163, 0.3076278865337372, -0.41537582874298096, -0.17650583386421204, -0.4513799250125885, 0.48296573758125305, 0.6138008236885071, 0.5470929741859436, 0.662358820438385, -1.5090065002441406, 2.0193638801574707, 1.019300103187561, 0.6488455533981323, 0.24755769968032837, 0.1315346211194992, -0.2465936541557312, -0.5479799509048462, 0.6902561783790588, 0.7232106328010559, -1.1466503143310547, 1.0395011901855469, 2.361029624938965, -0.18545235693454742, -0.0005597859853878617, -0.0526774525642395, -0.6149797439575195, -0.4637656807899475, -0.3565737009048462, 0.7166457176208496, 0.6911649703979492, -0.06229516118764877, 1.9517897367477417, 0.46970388293266296, -1.780127763748169, 0.9546454548835754, 0.2759554982185364, 0.6560153365135193, 0.2522072196006775, -0.10285355150699615, 0.5137595534324646, 0.4923113286495209, -0.5757153630256653, 0.8005957007408142, -0.278767466545105, 1.5228588581085205, 0.7265550494194031, -0.004818108398467302, -0.2124778777360916, -0.815386176109314, 0.4947359263896942, -0.7275760769844055, 1.1038926839828491, 0.2268362045288086, -0.016672857105731964, -0.25657424330711365, -0.3947921097278595, 0.48044562339782715, -0.43731263279914856, 0.013176637701690197, -1.335193157196045, -0.5434468388557434, -0.3585931956768036, 1.500702142715454, -0.8269664645195007, 1.0689359903335571, 0.34375935792922974, 0.04865088313817978, 0.8606864213943481, 0.2529594898223877, 0.00541736138984561, 1.0057666301727295, 0.6288129687309265, 0.8427199721336365, -1.3855955600738525, -0.5664515495300293, -0.809830904006958, -0.780062198638916, 0.8492355942726135, -0.9916645288467407, 2.0550780296325684, -1.0559219121932983, 0.2511267066001892, 0.5984691977500916, 0.3571026921272278, 1.7315988540649414, -0.8342679142951965, -0.5976235270500183, -0.3609387278556824, 0.8141306042671204, -0.19836688041687012, -0.4907199442386627, -0.4260735809803009, 1.1791515350341797, -0.4560158848762512, 0.20303520560264587, -0.47220996022224426, -1.1918882131576538, 0.444927453994751, 0.03644877299666405, -1.0946773290634155, -1.0964300632476807, 1.4128206968307495, 0.5715598464012146, -1.070056676864624, 0.8525946736335754, 1.058439016342163, 0.6485729813575745, 1.3319810628890991, 0.015554660931229591, -0.3193167448043823, 0.22489209473133087, 2.1021535396575928, -0.2587538957595825, 1.892904281616211, 0.020762478932738304, 2.3160088062286377, 1.6033823490142822, 1.922658920288086, 0.9855893850326538, -0.05055965110659599, -0.09653617441654205, 0.8940767049789429, -0.03113461285829544, 0.6510984301567078, -0.30473941564559937, 0.3181896507740021, 0.31058627367019653, -0.8516709208488464, -0.9816173911094666, 0.53046053647995, 1.3815889358520508, 0.21318776905536652, -1.2738035917282104, -0.5628786087036133, -1.2984129190444946, -0.8690654635429382, 0.502909779548645, -1.0542967319488525, -2.442821979522705, -1.5722761154174805, -1.4630213975906372, 1.5678844451904297, -1.790663719177246, -1.071913480758667, -0.3427777588367462, -0.44724738597869873, 1.320540189743042, -0.9955267310142517, -0.9670186042785645, -1.11004638671875, -0.5102742314338684, 0.0032167015597224236, -1.3036134243011475, -0.11769283562898636, 1.195678472518921, -0.6623775959014893, -0.8978564143180847, 1.006649374961853, 0.03487353026866913, 0.8034934401512146, 0.9696645140647888, 1.7647234201431274, 0.317041277885437, 1.604193925857544, 1.3843414783477783, -1.4947843551635742, -0.060711488127708435, 0.1572934091091156, 0.6678078174591064, 0.08505556732416153, 1.0911122560501099, -0.8249853253364563, 0.8283014297485352, 0.7735001444816589, 1.1971781253814697, 0.24561825394630432, 0.8141366243362427, 0.7057294845581055, -1.3575425148010254, 0.2012762725353241, 1.1424800157546997, 0.11412100493907928, 0.24706648290157318, -0.002519646193832159, 0.05741271749138832, 0.4656533896923065, 0.0697723925113678, 1.1239066123962402, 1.0371508598327637, 0.5513072609901428, -1.1562869548797607, 0.05770822986960411, -1.4258315563201904, 1.0882854461669922, -0.8058751821517944, 0.1527789980173111, 0.03141377121210098, -1.5470560789108276, 1.5964754819869995, 0.5968713164329529, -0.5724433660507202, 0.8696193099021912, 0.09114187210798264, -1.0877795219421387, -1.399507999420166, 0.35454246401786804, -0.461546927690506, -0.3062135875225067, 0.6634222269058228, 1.3692471981048584, -1.6547397375106812, 0.6392527222633362, 0.2601205110549927, 0.5370067358016968, 0.3588745892047882, 0.3977374732494354, 1.6853313446044922, -0.16487747430801392, -1.3759125471115112, 1.0339192152023315, 0.6213820576667786, 0.643642008304596, 0.35582295060157776, 0.2324979454278946, 0.5506559610366821, 0.8696914911270142, 0.08799443393945694, 1.546093225479126, -0.3871065378189087, 0.2567886710166931, 1.0062692165374756, 1.1492433547973633, -1.1942384243011475, 1.1776416301727295, -0.45502549409866333, -0.13949540257453918, -1.1215134859085083, 0.20349232852458954, -2.389622211456299, 0.2540438771247864, 0.5092830657958984, -0.7901861071586609, -0.12120195478200912, 0.7323845624923706, 1.1398351192474365, 0.13812725245952606, 1.0387059450149536, 0.2346305549144745, -0.9212412238121033, -0.49222874641418457, 1.5858136415481567, 0.6103202700614929, 2.1006336212158203, 0.7436032891273499, 0.012406094931066036, 1.5672862529754639, 0.8047757148742676, -0.5029430985450745, 0.1662846952676773, 0.9792487621307373, -0.708609402179718, -0.9233192205429077, 0.9497604370117188, 0.5894434452056885, -0.505155086517334, -1.3986691236495972, 1.4693529605865479, -1.1082992553710938, 0.27426692843437195, 1.7638463973999023, -0.8337687849998474, 0.09806405007839203, -1.1617431640625, 0.5115240812301636, 0.17194847762584686, 1.3053209781646729, -0.5280365347862244, -1.4549261331558228, -1.263420820236206, -0.5557258725166321, -1.1135330200195312, -0.32707804441452026, -0.5686381459236145, -2.151864767074585, 2.3180770874023438, 2.450350284576416, 2.2146639823913574, 0.4884873628616333, 0.16397801041603088, 1.2295299768447876, 0.5352891087532043, 0.7539924383163452, 0.8931928277015686, 0.047470755875110626, 0.5903101563453674, -0.057713720947504044, -0.3923652172088623, 0.19281324744224548, -1.4641557931900024, -0.7737560868263245, 0.2835557460784912, -0.33459460735321045, 1.2599910497665405, -1.000840187072754, -0.5496793985366821, -0.3798793852329254, -1.0518622398376465, 1.2564023733139038, 0.39162716269493103, 1.9662524461746216, 0.17439012229442596, 0.557204008102417, 0.02304023690521717, 0.11302754282951355, -0.9362363219261169, 0.4363284707069397, -0.45461562275886536, 2.3011362552642822, 0.969786524772644, 0.7719416618347168, -0.25441697239875793, 1.177546501159668, -1.5538012981414795, 1.5442570447921753, -0.029788244515657425, -0.7281846404075623, 0.002364020561799407, 0.16044718027114868, 0.1610942929983139, -1.4862141609191895, 0.38593077659606934, 1.438603162765503, 0.2028120756149292, 0.6604012250900269, 0.9120965600013733, 1.039066195487976, 0.9611194729804993, 0.4172556698322296, -0.9159397482872009, 1.6822738647460938, -1.4704941511154175, 1.4283733367919922, -0.16213680803775787, -1.6415082216262817, 0.8315823674201965, -0.9791478514671326, 0.5045576095581055, 0.30329105257987976, -1.5938127040863037, 1.249796986579895, 0.6158837080001831, -1.0351388454437256, 0.14668767154216766, 0.0029371539130806923, -0.784062922000885, -1.1826704740524292, 0.39440789818763733, 0.20977799594402313, -0.5563777685165405, 1.4874413013458252, 0.15195518732070923, -0.3150513768196106, 0.9403297305107117, -0.4632992148399353, 1.3211688995361328, 0.5051966309547424, 1.1780027151107788, 1.0044872760772705, 1.95348060131073, -1.5549567937850952, -1.5022245645523071, 0.5515905022621155, 0.8841400742530823, 0.7108004689216614, -1.9804075956344604, 0.11070937663316727, -0.6856420636177063, 0.29239940643310547, 0.5690740346908569, 0.3180480897426605, 0.034998342394828796, -2.189699172973633, -1.2352608442306519, -0.3603450655937195, 2.3427574634552, 0.014181023463606834, -1.8653284311294556, -0.9872939586639404, -0.17208927869796753, -0.5174397826194763, 2.7913639545440674, 0.9368269443511963, -0.13795582950115204, 1.3285191059112549, 0.9303355813026428, -0.9807482361793518, 1.5257654190063477, 0.7487147450447083, -0.27470967173576355, -0.2589825987815857, 0.9296483993530273, -0.9582242369651794, -0.753349244594574, 1.1645008325576782, -0.20381373167037964, -0.07936792820692062, -0.6868279576301575, -0.06461048126220703, -0.17283642292022705, -1.0502411127090454, 0.7813305258750916, -0.13519397377967834, -1.4709292650222778, 0.9005621671676636, 0.37165865302085876, -1.4499276876449585, -1.45290207862854, 0.5637286901473999, 0.24072176218032837, -0.46582719683647156, -2.0278475284576416, 0.8632094264030457, -0.9648090600967407, -0.9468031525611877, 0.09828278422355652, 0.45078539848327637, 1.1045857667922974, 0.29508644342422485, -0.25333860516548157, -0.5560517907142639, -0.46585431694984436, 1.1860325336456299, -1.6278119087219238, -0.030065331608057022, -0.08253734558820724, -1.8238321542739868, 1.530274748802185, 0.07447788864374161, 2.10209321975708, 1.299843192100525, -1.3313026428222656, -0.6038427352905273, 0.9212431311607361, 1.8590433597564697, -0.00886213406920433, -0.7690843343734741, 0.01549911592155695, -0.6369367837905884, 0.08391828089952469, -0.9289360046386719, -1.0381981134414673, 0.6214733719825745, -0.6109726428985596, 0.6895896196365356, 0.3593003451824188, 0.8322548866271973, -1.6603690385818481, -0.1308448761701584, 0.1825142204761505, -0.6748930811882019, 1.153638243675232, 0.6633878350257874, 0.7811672687530518, 1.4086518287658691, -0.0032130221370607615, -0.1825488954782486, 0.13960622251033783, -0.8235135078430176, 0.9253444671630859, -0.3230588138103485, 0.28647688031196594, 1.6591163873672485, -0.7215859889984131, -0.4948584735393524, 0.7155348658561707, -1.1326253414154053, -0.4941374957561493, -1.3471660614013672, -1.5224400758743286, -0.3498251736164093, 0.5731749534606934, 0.1353309154510498, -1.9124982357025146, -0.22436180710792542, 0.27497774362564087, -1.2444700002670288, 0.3139571249485016, -1.199922800064087, -3.0265567302703857, 0.5625189542770386, -0.9428050518035889, 0.01988992840051651, -0.6761094927787781, 0.6207898855209351, 1.8142601251602173, 0.26413169503211975, 0.08024569600820541, 0.5848598480224609, -1.0400162935256958, -0.31445425748825073, 0.33990219235420227, -1.4448333978652954, 0.13741765916347504, 1.3474351167678833, -0.1694372296333313, 1.6722744703292847, -0.12012884765863419, -0.09371558576822281, 0.06827374547719955, -0.9584167003631592, -1.0961008071899414, 0.7528998255729675, 0.3311663568019867, 0.24133729934692383, -1.152055263519287, 0.6307715177536011, -0.20498408377170563, 1.2585324048995972, -0.41126352548599243, -0.380927175283432, -1.2030738592147827, -0.18838952481746674, 0.07089600712060928, -0.6111360788345337, -0.48007333278656006, 1.6472089290618896, 1.058466911315918, -1.0751246213912964, 1.3205602169036865, 0.5839533805847168, -0.0065752193331718445, -1.1469019651412964, -0.2729181945323944, -0.14264848828315735, -0.03952080011367798, 0.39282625913619995, 0.4898845851421356, -0.15820644795894623, 1.4814562797546387, 0.3694763779640198, 0.06425616890192032, -0.3987904191017151, -0.9441691040992737, 0.57767254114151, 0.09511269629001617, -0.8001518249511719, -0.49884551763534546, 0.8542274832725525, -1.3705477714538574, -1.3780933618545532, 0.6504606008529663, -1.2564070224761963, -0.42573264241218567, 1.349960207939148, -0.49963808059692383, 0.718887984752655, 0.13268691301345825, -0.6976773738861084, 0.1793287992477417, -0.8364484906196594, -1.33258855342865, -1.4213758707046509, -0.6878744959831238, -1.7158907651901245, -0.7943341732025146, -0.1480129510164261, -0.4021523892879486, 1.4636518955230713, 2.0597686767578125, -0.9631037712097168, 0.8252298831939697, 0.5749709010124207, 0.6751335859298706, -0.537224531173706, -0.7179808616638184, -0.559859573841095, 0.6213669180870056, -1.8181930780410767, -0.3075654208660126, -0.5844271183013916, 0.6208797097206116, -1.0696865320205688, 0.393136203289032, 1.348953366279602, 0.28372177481651306, -0.0014486846048384905, 1.8788046836853027, 0.5580909848213196, -0.378491073846817, 0.32673001289367676, 0.716355562210083, -0.4382678270339966, -0.14045049250125885, -1.531325101852417, 0.03601808100938797, -0.8765758275985718, -0.0627501979470253, 2.4237263202667236, 1.8653427362442017, -2.1874420642852783, 1.760401725769043, -0.31724587082862854, 1.4547115564346313, -0.9850356578826904, -0.3195037543773651, -0.14556178450584412, 0.8984758853912354, -0.07548325508832932, 0.9761066436767578, -0.49023038148880005, -0.007801420520991087, 0.05292883142828941, -1.3358110189437866, 0.03972986713051796, 0.0712105929851532, 0.28963983058929443, 0.0669684112071991, -1.2255886793136597, -0.2838188707828522, 1.2930607795715332, -0.45185571908950806, -0.5643857717514038, -2.1701252460479736, -0.723543643951416, -0.19417177140712738, 0.04857894778251648, 1.065236210823059, -0.34620004892349243, 0.46564292907714844, 0.873908281326294, -0.41620662808418274, -0.24775423109531403, -0.505084753036499, 0.01463567465543747, -0.5703877210617065, -0.2250543236732483, 0.7848013639450073, 1.6740952730178833, 1.4007521867752075, 0.6028079986572266, -1.3626790046691895, -0.219954714179039, -0.931318998336792, -0.09966474026441574, -0.2793824076652527, -0.43359747529029846, -0.1785033643245697, 0.3195933401584625, 2.104612112045288, -0.7979663610458374, -0.1766667515039444, -0.35154053568840027, 0.010276357643306255, 0.23879112303256989, -0.5258166790008545, 2.169837713241577, 0.06081899255514145, 0.744539737701416, -0.6697450876235962, 0.9113051295280457, -0.7448910474777222, 0.3286167085170746, 2.1885170936584473, 1.508541226387024, -0.600271463394165, -1.621554970741272, -1.757121205329895, 0.07290398329496384, -1.6965845823287964, 2.127321720123291, 1.7771705389022827, -1.25740647315979, 1.1301450729370117, -1.5229063034057617, 0.1669577956199646, -0.3792784512042999, 2.3964388370513916, 0.12559670209884644, -1.4076931476593018, 1.1404669284820557, 0.993201494216919, 0.5802653431892395, -0.38365626335144043, -2.2476260662078857, 0.38587403297424316, -0.7089821100234985, 0.09521342813968658, -1.1805747747421265, 1.3538124561309814, 0.7953813076019287, -0.5393550992012024, 0.9741780757904053, -0.9189175367355347, -0.4840140640735626, -1.0104252099990845, -2.18672251701355, -0.643388569355011, 0.7204250693321228, -0.5266279578208923, 0.21130746603012085, 1.0061683654785156, 0.3654901385307312, -0.15034477412700653, 0.09720712155103683, 0.7251104116439819, -0.16042132675647736, -1.2568989992141724, 0.9524245858192444, -1.4349150657653809, -1.438521146774292, -1.370269536972046, -0.21913398802280426, -0.42983314394950867, -0.10753355175256729, 1.9064548015594482, -1.3747628927230835, -0.8266510963439941, 0.7341524958610535, 0.8003829717636108, 0.5297303795814514, 0.392233282327652, -0.5539077520370483, 1.5983080863952637, -1.2279558181762695, 0.7923756241798401, -1.437838077545166, 0.9355340600013733, -0.15564726293087006, 1.3972562551498413, 0.31325143575668335, 0.3374391198158264, -1.0655075311660767, -1.0672019720077515, -0.2527027726173401, -0.33499887585639954, 0.17845039069652557, 0.2788446545600891, 0.6660888195037842, -1.0837510824203491, 0.194738507270813, 0.44053399562835693, 0.1962066888809204, -0.046744368970394135, 1.0904499292373657, 1.2514649629592896, -0.2547413408756256, 0.31711193919181824, -0.34568583965301514, -1.8679676055908203, -2.6388027667999268, 0.0637504830956459, -0.2317972332239151, -0.5326523184776306, -0.1353030949831009, -0.6216354370117188, -1.1210265159606934, 1.1701157093048096, -0.04867221415042877, 0.1105465367436409, -1.5775535106658936, 0.2862628400325775, 0.7721051573753357, -0.3749806880950928, -1.0203474760055542, 1.8904752731323242, 1.7853237390518188, -0.007848132401704788, 0.9659733176231384, -0.6923018097877502, 0.2277047485113144, -1.7623623609542847, -0.18392562866210938, -3.4515676498413086, -0.22329631447792053, -0.1612090915441513, 1.1496511697769165, -1.3120638132095337, -0.3723997175693512, -0.32966580986976624, 2.640137195587158, 0.9620383977890015, 1.5196820497512817, 0.211202934384346, 1.1742968559265137, -1.1259613037109375, -0.7871928215026855, -1.9126538038253784, -0.7591121196746826, 0.712478756904602, -0.43654555082321167, 0.17963768541812897, 0.03056606464087963, 0.35098981857299805, -0.9516141414642334, 0.45497965812683105, 0.5077729821205139, -0.7558237910270691, 0.08266160637140274, -0.35948339104652405, -0.11665627360343933, 0.5734826326370239, 1.538414478302002, 0.4219091236591339, -0.2778632938861847, -1.040861964225769, -0.31060904264450073, 0.2335285097360611, -0.2845704257488251, 0.08941840380430222, 0.5216676592826843, 0.835564374923706, -1.274106502532959, -1.1782881021499634, 0.9400146007537842, -1.8727988004684448, -1.9071100950241089, 1.2694706916809082, 1.3754864931106567, -0.6870363354682922, -1.0974860191345215, 1.186956763267517, 1.4669729471206665, 0.7964271903038025, -0.9655934572219849, -0.17772915959358215, -1.00399911403656, 1.9721592664718628, -0.5514316558837891, -1.8355650901794434, 1.1851977109909058, -0.2110602855682373, -0.35875949263572693, 0.7059959769248962, 0.4159211814403534, -0.4745994210243225, -1.4839636087417603, -0.1291555017232895, -1.1387419700622559, 0.9546403884887695, -0.16398273408412933, 1.8412266969680786, -0.35994645953178406, 0.634272038936615, -0.06555356830358505, -0.4502485394477844, 0.8122780323028564, -1.760096788406372, 0.23677600920200348, 0.13982896506786346, -0.4354303181171417, 1.5514352321624756, -1.2203675508499146, -1.1866406202316284, 0.3149290382862091, -1.4381027221679688, -1.0467265844345093, -0.15466248989105225, -1.175683617591858, -0.39046746492385864, 1.199584722518921, -0.1284560263156891, 1.6837631464004517, 0.37561336159706116, 0.35466793179512024, -0.8574619889259338, -1.2214151620864868, 0.22160305082798004, -1.0210040807724, -1.2822712659835815, -2.1407382488250732, 0.9481566548347473, 0.3248434066772461, 0.9705570340156555, 1.1789618730545044, -0.44030946493148804, 2.406862258911133, 0.9656307101249695, 1.165549635887146, -1.4482486248016357, 0.9454390406608582, 0.22005484998226166, -1.178933024406433, -0.8152695298194885, -0.6703149080276489, 1.1869782209396362, 0.050585292279720306, -0.44408315420150757, -1.8657606840133667, -0.0282431673258543, -0.17152224481105804, -0.2854028344154358, -0.4417816996574402, 1.2179648876190186, -0.11204387247562408, 1.1297341585159302, 0.8401198387145996, -0.08979486674070358, -0.2911839187145233, 1.3104430437088013, -0.9950224757194519, 1.0238062143325806, -0.6346397399902344, 0.5143091678619385, 1.6275144815444946, -2.0843865871429443, -0.586867094039917, -2.8025565147399902, 1.5504378080368042, -1.189400553703308, 0.29474586248397827, -0.6389601826667786, 1.5043941736221313, -0.11037413030862808, 0.14680488407611847, -0.81849604845047, -0.14437857270240784, 0.291166752576828, 0.8018569350242615, 0.9296057820320129, 1.1778866052627563, 1.1000245809555054, -2.0736944675445557, -1.402869462966919, -0.797559916973114, 0.680962324142456, -0.6626229882240295, 1.4539767503738403, -0.6017888784408569, 0.6409284472465515, 1.6318104267120361, 0.7298070192337036, 0.1988515704870224, -0.7840365171432495, 1.5216591358184814, -0.22454100847244263, 0.976689875125885, 0.3764895498752594, -0.004304543137550354, 0.24624159932136536, -1.2470232248306274, 0.20244282484054565, 0.40764373540878296, -1.3452757596969604, 0.8013100028038025, -0.6782429218292236, 0.11376223713159561, -0.6711097359657288, -1.4271281957626343, -1.5549920797348022, 0.31232506036758423, -0.33533743023872375, 1.7857544422149658, 0.724390983581543, 0.025109196081757545, 0.06534823775291443, -2.75748610496521, 0.5481959581375122, 0.27889543771743774, -0.2914167642593384, -0.7367836833000183, -0.23320220410823822, 0.6314675211906433, 0.03656936064362526, 1.3670052289962769, -0.6111721992492676, -1.5704385042190552, 2.034269094467163, -0.25972071290016174, -0.2492494136095047, 0.12541235983371735, 0.576989471912384, -0.9917734861373901, 0.245205357670784, -3.346506357192993, -0.03238280862569809, -0.43852508068084717, 0.9879238605499268, 0.07859542220830917, -0.5757644176483154, 0.10599830746650696, 0.4831230044364929, 1.220172643661499, -1.1748690605163574, 0.31879544258117676, 0.25602152943611145, 0.952845573425293, 0.04666896536946297, -0.6884890198707581, 0.9579737782478333, -0.5000981688499451, 0.020789993926882744, -0.3826545774936676, -0.41477492451667786, -0.9513430595397949, -1.0660429000854492, 0.9432655572891235, -0.8187971711158752, -0.05369766801595688, -0.41465944051742554, -0.5507245063781738, 0.7564244866371155, 0.29880663752555847, -1.1126614809036255, 0.4178455173969269, -1.543375015258789, 1.5832536220550537, 0.9614851474761963, -0.24375838041305542, -0.3571067750453949, -1.9009959697723389, 0.8210284113883972, 0.22516053915023804, -0.3808675706386566, 0.03493574261665344, -0.8293386697769165, -0.9346849322319031, -0.7655150294303894, -1.5284639596939087, -1.6048476696014404, -0.5321706533432007, -0.21967919170856476, -0.16719795763492584, 1.4973522424697876, -0.7476811408996582, 0.9298915266990662, 0.6336853504180908, 1.657236933708191, -1.1926720142364502, -1.1937220096588135, 0.849494218826294, -0.41479769349098206, 0.002981010125949979, -1.3304318189620972, -0.2651335597038269, -0.8590671420097351, -0.7277838587760925, -1.430084466934204, -0.6594315767288208, 0.6281441450119019, -1.1882431507110596, 0.4515461325645447, -1.4578205347061157, -0.16102585196495056, 0.25746726989746094, 0.2334524542093277, 0.1283249408006668, -0.0974418893456459, -0.9616467356681824, 0.8443239331245422, 0.5391766428947449, -0.3565944731235504, 0.8140659332275391, -0.17987248301506042, 0.46669110655784607, -0.32342177629470825, -1.3056321144104004, -1.745882511138916, 0.20759934186935425, 0.41832688450813293, -0.767524778842926, 1.442028522491455, -0.7660563588142395, 0.811034083366394, -1.0379761457443237, 2.063089609146118, -0.39734354615211487, -1.943359375, -1.36630380153656, 0.14690178632736206, -0.06260994076728821, 1.2187062501907349, 1.6507576704025269, -0.4393892288208008, -1.0257774591445923, 1.1424967050552368, 1.6805698871612549, -1.5041751861572266, 1.1722403764724731, -0.36024779081344604, -0.2426263988018036, 0.32375526428222656, 1.0396947860717773, -1.6790493726730347, -0.9291316270828247, 1.2975224256515503, -0.2705053389072418, -0.9455578923225403, -0.17267511785030365, 0.6474298238754272, 0.11305616050958633, -0.9187275767326355, 0.2042904794216156, 0.07069215923547745, 0.02287529595196247, 0.889405369758606, 0.1582961529493332, -2.4215614795684814, 0.4582049250602722, 0.004408445209264755, 0.3827558159828186, -0.17120039463043213, -0.008439842611551285, -0.672515332698822, -0.3237001597881317, 0.9305200576782227, -1.7105664014816284, -1.5057191848754883, 0.680525004863739, -0.7638307213783264, -0.4428582489490509, -1.3168163299560547, 0.450924813747406, -2.032379150390625, 0.4054237902164459, 0.4772263467311859, -0.5649629235267639, -0.7174929976463318, -0.3552539050579071, 0.9542151093482971, 3.1305675506591797, -0.036714471876621246, -0.5467041730880737, -0.46616074442863464, -0.38891246914863586, 0.5519607067108154, -0.20731814205646515, 0.9378006458282471, -1.0043354034423828, 0.4783858060836792, -0.20973439514636993, -1.518998384475708, -0.1337793916463852, -0.370916485786438, 1.149941086769104, -0.031058041378855705, -0.47529351711273193, -0.7867670059204102, -1.5443569421768188, -0.3793649673461914, -0.03534566983580589, -0.5027581453323364, -1.410603404045105, -0.15143738687038422, -1.3736908435821533, 1.6002544164657593, 0.6088444590568542, -1.1238505840301514, 1.244046926498413, 0.11328797042369843, -0.44918084144592285, -0.1804984211921692, -0.4564986824989319, 0.09149486571550369, -0.4856133460998535, -1.4661250114440918, 0.8638831973075867, 1.2649919986724854, -0.6553227305412292, 0.10861734300851822, -0.2511797547340393, -0.9976608157157898, -0.864416241645813, -0.3513903319835663, -1.6220176219940186, -1.0206654071807861, -0.10228808224201202, 0.3384644687175751, 0.2964925765991211, -0.362286239862442, -0.45349830389022827, -1.0781919956207275, 0.16307124495506287, 0.9475436210632324, -1.2797728776931763, 2.26210355758667, -1.7104958295822144, -0.8031005859375, 1.73409903049469, -0.5439985990524292, -0.570374608039856, 0.5815718173980713, 1.1055057048797607, -0.2468787431716919, -0.058031219989061356, -1.9148328304290771, 1.006676197052002, -0.9578269720077515, -1.8496646881103516, 0.040976185351610184, 0.7588549256324768, 1.5995476245880127, -1.3270938396453857, 0.5373482704162598, 0.322894424200058, 1.7595067024230957, 0.3251727819442749, -0.312035471200943, -0.4312276542186737, -0.20301580429077148, 0.3913979232311249, -0.6430633664131165, 0.08254636079072952, -0.8986196517944336, -0.3838600814342499, -0.2479492723941803, 0.6461275815963745, -1.6867982149124146, -1.4889070987701416, -0.6672329306602478, -0.4003036618232727, -0.20957563817501068, 1.0220390558242798, -3.7148783206939697, -0.7124618291854858, 0.7496824264526367, -0.40161481499671936, -0.017424549907445908, 1.2792586088180542, 1.0339369773864746, -0.17299200594425201, -0.42351773381233215, 0.5683449506759644, 2.9440386295318604, 0.15226447582244873, -1.6008291244506836, 0.2848564088344574, 1.2861275672912598, -2.0878593921661377, 1.3140326738357544, -1.7279648780822754, 0.7193009257316589,0.047717247158288956, 0.21084433794021606, 0.4612586498260498, 0.6689397692680359, 1.5978376865386963, 0.7506820559501648, 0.3051967918872833, 0.948640763759613, 0.9415479898452759, -0.7831985354423523, 2.1292524337768555, -0.6728421449661255, -1.136971116065979, 1.0478206872940063, 0.5777735710144043, -1.5553061962127686, 0.8038011789321899, 1.3289164304733276, 1.153518795967102, -2.0303401947021484, -1.9604051113128662, 0.3146425783634186, -1.2158859968185425, 0.04048111289739609, 0.02396119013428688, 0.8826922178268433, -0.18064147233963013, -0.15666237473487854, -0.12529072165489197, 0.16924211382865906, -1.6936371326446533, -2.1431825160980225, 0.19771283864974976, -1.2720927000045776, 1.396331548690796, -0.7063277959823608, -0.8545072078704834, -0.8222666382789612, -0.9081456661224365, 0.4777528941631317, 0.13193286955356598, 0.09540288150310516, -0.4275261461734772, -0.2724137008190155, 0.1377328783273697, 0.9301137328147888, -0.1060660108923912, -0.5717463493347168, 0.5705773234367371, 0.6820522546768188, -1.451804518699646, -0.7098889350891113, -1.1061925888061523, 0.3925279378890991, 0.8108583688735962, 1.1701120138168335, -0.284684956073761, 0.33812421560287476, 0.05065690353512764, -0.25125983357429504, 0.5168737173080444, -0.8554295301437378, -1.0772149562835693, 0.318108469247818, -1.259049654006958, -1.4725145101547241, -0.26740318536758423, 0.6285285949707031, 0.5141673684120178, -0.5636006593704224, -0.3620452582836151, -1.3173046112060547, 0.26414233446121216, -0.289590984582901, -2.096057176589966, -1.043019413948059, 0.6922906041145325, -0.3460937440395355, -1.1605952978134155, 0.29884442687034607, 0.13609272241592407, 0.8900149464607239, 1.2888811826705933, 0.7342342138290405, 0.24658335745334625, 0.8307083249092102, 0.49005720019340515, -0.5712683796882629, -0.2702498733997345, -1.3671776056289673, 0.6985225081443787, 0.7046346068382263, -0.26276740431785583, 0.5768918395042419, 0.6281687617301941, 0.8663529753684998, -1.7284075021743774, -0.4995538592338562, 0.436858594417572, -0.3484940230846405, -1.201076626777649, 0.31453704833984375, -1.0518825054168701, 0.18744350969791412, 0.12909087538719177, 2.10178804397583, -0.5546457767486572, 1.2257214784622192, -0.7249008417129517, 0.38968655467033386, -1.4383970499038696, 1.129080891609192, 1.5353612899780273, 0.5795002579689026, -0.8314839601516724, 0.38964518904685974, -0.004885345231741667, 0.4793272912502289, -0.9192218780517578, 2.0837128162384033, -0.360291451215744, -2.0384018421173096, 1.6389288902282715, 0.11768332123756409, -1.1041491031646729, 0.47410330176353455, 0.7422705292701721, -0.27235183119773865, -0.6065242290496826, 0.9968979358673096, -0.18647056818008423, 0.2982662320137024, -0.82035893201828, 1.000251054763794, 0.42521655559539795, 0.6384099721908569, -0.4237314760684967, -0.24413298070430756, -0.0275197084993124, 1.1306065320968628, -0.4900984466075897, 0.8314799666404724, 0.5010020732879639, -0.8207727074623108, -0.784766674041748, 0.20642970502376556, -0.9799897074699402, -1.077019214630127, -0.5759512186050415, 1.0319832563400269, -1.2079074382781982, 1.2266416549682617, -0.2866867482662201, 0.906305730342865, 2.197190761566162, 0.7834852933883667, 1.4061740636825562, 0.24817724525928497, 0.15801605582237244, 0.6982513070106506, 0.3664604425430298, -1.03168785572052, -0.8162450194358826, -0.5166845917701721, -0.146807461977005, 0.23882660269737244, -0.023326629772782326, -0.5475584268569946, -0.07081031799316406, 1.6104854345321655, -0.19945000112056732, 0.10040334612131119, 1.3992738723754883, 0.22064130008220673, -0.13702034950256348, 0.5485379099845886, 1.0765093564987183, 1.1144567728042603, -1.050877332687378, -0.8257864713668823, 1.183901071548462, -0.47990718483924866, 0.25615817308425903, -0.032469235360622406, -1.2898037433624268, -0.7003337740898132, 0.428146630525589, 0.6354815363883972, 0.5272490978240967, -1.0385329723358154, -2.106009006500244, -0.29063811898231506, 0.2730545997619629, -0.3329741656780243, -0.34976181387901306, -1.1764938831329346, 0.31767362356185913, 1.7513400316238403, -0.32590019702911377, 0.18862658739089966, 1.218144416809082, 1.8267375230789185, -1.1202976703643799, -2.8970627784729004, -0.6018227934837341, 0.7844710946083069, -0.30946239829063416, -1.1893852949142456, -0.2831120491027832, -0.8874563574790955, 2.244614839553833, -1.4232131242752075, -1.3162976503372192, -1.0546867847442627, -0.36810627579689026, 1.0313156843185425, 1.405451774597168, -0.8249859809875488, 0.04638012498617172, 0.10364831984043121, 1.432382345199585, 0.07661547511816025, 1.9118518829345703, 0.3440001308917999, -0.4365920424461365, -0.04993794113397598, -0.02340543083846569, 0.1701379120349884, 0.7727062106132507, 0.3289715647697449, -0.5883509516716003, 1.342180609703064, -0.4542553424835205, 0.5303173661231995, 0.8540997505187988, 0.6606034636497498, -0.3052508234977722, 1.9676008224487305, -0.47212493419647217, -0.3630330264568329, -1.8373346328735352, -1.2871592044830322, -0.6957984566688538, -0.8461524248123169, 0.3935796916484833, -1.6806541681289673, 0.30488449335098267, -0.7303101420402527, -0.5395167469978333, 1.0349032878875732, -0.476683109998703, -0.5382452011108398, -0.09512051939964294, -0.3782208561897278, -1.1957224607467651, -0.3742212653160095, -0.517737865447998, 0.006710302550345659, 0.19567209482192993, -0.7689359188079834, 0.608939528465271, 0.8243175745010376, -0.7450968027114868, -0.12146145850419998, -0.3290347158908844, -0.9731864929199219, -0.8426527380943298, 0.7637643814086914, -1.6719094514846802, -1.5145665407180786, 0.5349875092506409, 0.3397526144981384, 0.21333451569080353, 0.24570989608764648, -1.1619549989700317, 0.753006100654602, -0.04784731566905975, 0.30519038438796997, 0.6939603686332703, 0.41752979159355164, 0.5426839590072632, 0.6476499438285828, -1.4261689186096191, 1.9166179895401, 1.050794243812561, -1.7101848125457764, -1.3130784034729004, 1.6864371299743652, 0.7490285038948059, -0.15160050988197327, -1.6802572011947632, 1.1262261867523193, -0.7739332318305969, -0.8766727447509766, 0.6197783350944519, 1.9047861099243164, 1.8700674772262573, -0.8559051156044006, -0.06141619756817818, 0.8898953199386597, 0.8980222940444946, 0.8732943534851074, -0.4113747179508209, 0.1287514865398407, 0.3847011923789978, -0.5700910091400146, -0.5010945200920105, 0.6654459834098816, 0.4297395348548889, -1.3112075328826904, -0.9092803001403809, -0.6493514180183411, 0.43000656366348267, 0.9589614868164062, 1.1050851345062256, 0.16769607365131378, 2.0569698810577393, -0.28817859292030334, 0.3040761947631836, -0.7773591876029968, -0.8220275640487671, -0.32229384779930115, -2.4082045555114746, -0.42596930265426636, -0.3753955364227295, 0.38148394227027893, 1.2677284479141235, -1.040696620941162, -0.01125628873705864, 1.1446284055709839, -0.976384699344635, 0.7003414630889893, -1.212842583656311, -0.7472978830337524, 0.015023235231637955, -0.729748547077179, 1.8760863542556763, -0.38933148980140686, 0.10551991313695908, 0.36704784631729126, 1.4671571254730225, 0.33303117752075195, -0.1992732435464859, 0.5365217924118042, -0.23077233135700226, -0.06241803616285324, -0.08018025010824203, -0.36694613099098206, -1.745110034942627, -1.1914905309677124, 0.5744803547859192, -0.11334766447544098, 0.526075005531311, -0.5739768743515015, -0.8178113698959351, -1.5546799898147583, 0.8174230456352234, 0.39463117718696594, 0.003207228612154722, 0.35531988739967346, 0.6468367576599121, 0.20784230530261993, -0.2969632148742676, -0.11683650314807892, 0.2141849845647812, 0.7103276252746582, 0.4196541905403137, -0.006321470253169537, 1.5024878978729248, 0.8179230690002441, 1.3664096593856812, -0.4490884244441986, 0.7767404913902283, -1.938584566116333, -0.5679614543914795, -0.8598304986953735, -0.13767853379249573, 1.1780461072921753, -1.3220363855361938, -0.22753114998340607, 0.9559422135353088, 0.15691456198692322, -0.15103904902935028, -2.0192065238952637, 0.8014703392982483, -0.4993681013584137, 1.62299382686615, -0.988615870475769, -0.841666579246521, -2.199232339859009, -1.569582462310791, 0.12987107038497925, -0.7131416201591492, 0.07584966719150543, 0.10627351701259613, 0.6596389412879944, 0.3108394742012024, 2.24556827545166, 0.5975973606109619, -0.21387960016727448, 1.1463868618011475, 1.3943307399749756, -1.8681703805923462, -0.6250263452529907, -0.5385173559188843, -0.4880625903606415, -0.08256864547729492, 0.8475365042686462, 0.22341912984848022, 0.519683837890625, 0.30028054118156433, 0.7540889382362366, 0.8428552150726318, -0.7166087627410889, 0.8898228406906128, -0.10957107692956924, 0.000872215663548559, -0.8654459714889526, -0.10847315192222595, -0.230314239859581, 1.7378296852111816, 0.26275232434272766, 1.576568603515625, -0.03477667272090912, -1.2121526002883911, -0.6022401452064514, -2.1508705615997314, 0.10254281014204025, -0.12530048191547394, -0.004957541823387146, -0.25380587577819824, 0.9464564323425293, 0.26776427030563354, -0.0009171689162030816, 0.4651618003845215, 0.39452385902404785, 1.8090076446533203, 0.20383965969085693, 0.6719174385070801, 2.0374915599823, 1.1461971998214722, 0.5916748046875, 0.553226888179779, 1.309985876083374, -0.02114388532936573, -0.7130926847457886, 0.6871941089630127, 0.7368391156196594, 0.3158739507198334, -0.6529485583305359, 0.20490194857120514, -0.9292020201683044, 0.668921947479248, -1.3303284645080566, 0.9392021298408508, -0.20357316732406616, 0.3274862766265869, -0.26285797357559204, 1.7256184816360474, 0.4934380352497101, -0.2681131660938263, 0.7576808929443359, -0.4032219648361206, 0.6603093147277832, 1.0555713176727295, 1.6023792028427124, 2.0468835830688477, -0.8664820194244385, 1.0067315101623535, -0.05035967379808426, -1.694220781326294, 0.40733858942985535, 0.10005040466785431, 0.24118134379386902, -0.44232630729675293, 0.4777008891105652, 1.582152009010315, 1.106212854385376, 0.15487009286880493, 0.9007404446601868, -0.9174634218215942, -0.6128351092338562, -0.6770175099372864, 1.6515448093414307, 0.9908477067947388, 0.9821813702583313, -0.6781676411628723, -0.5334492325782776, 0.9723129868507385, 0.7119137644767761, -0.19187487661838531, 0.28808650374412537, 0.8314037322998047, -0.1683080643415451, -1.2078944444656372, -0.8615795373916626, 0.16745848953723907, -0.6006065607070923, -0.848258376121521, 0.1685682237148285, -0.7736362218856812, 0.11178720742464066, -0.007557492703199387, -0.38927531242370605, 1.9803273677825928, -1.0302414894104004, 0.7194711565971375, -0.45057085156440735, -0.4932941198348999, 1.0506070852279663, 0.23223596811294556, 0.5675492882728577, -1.2473245859146118, 0.9766266345977783, -1.1588233709335327, -0.5846466422080994, 0.4164256155490875, 1.652385950088501, 0.7306568622589111, -1.6145199537277222, 0.3193033039569855, 0.2775479853153229, 0.8121586441993713, 0.02584065869450569, 1.042441487312317, 0.14602690935134888, -0.2990158796310425, 1.8203696012496948, -0.3116937577724457, -2.1334457397460938, 0.4997944235801697, -0.5847301483154297, 1.6440354585647583, -1.9476287364959717, -1.0473471879959106, 0.7910942435264587, -1.0998344421386719, 1.2283896207809448, -1.103541374206543, -0.9594653248786926, -0.43149468302726746, 1.1310921907424927, -0.1298435479402542, -0.9957665205001831, 0.44931408762931824, 0.13985979557037354, 0.2378029227256775, -0.12390261888504028, 0.40394508838653564, -1.8064199686050415, -0.4742777645587921, 2.081685781478882, 0.2280692160129547, -0.9489055275917053, -0.6273154020309448, -1.0649992227554321, 1.1316676139831543, 0.7868269085884094, -0.9472208619117737, -0.8316314220428467, -0.7871355414390564, 0.8650702238082886, 0.07525940239429474, 0.4394930303096771, -2.160771608352661, 0.7532727122306824, 0.306301087141037, 0.12323612719774246, -0.7914727330207825, -0.064998559653759, -0.32140934467315674, 1.3491101264953613, 2.02233624458313, -0.09101278334856033, 1.8158053159713745, -0.6722787618637085, -0.5124808549880981, -1.3248865604400635, 0.01129512395709753, -0.1003236398100853, 0.15396183729171753, 0.35333821177482605, 0.08831045776605606, 0.6161710023880005, 1.1976234912872314, 0.44514843821525574, 0.4194369614124298, -0.5730845928192139, 1.4346673488616943, -0.595306932926178, 0.4008648991584778, 0.36129236221313477, 0.5740880966186523, 1.3881367444992065, -1.7001866102218628, 0.3199371099472046, 0.29879429936408997, 1.147035002708435, -1.9905673265457153, 0.4579874873161316, 0.7844094634056091, 1.9488939046859741, -0.21420635282993317, 0.9038315415382385, -0.4387907385826111, 1.3173471689224243, 0.6045706868171692, 0.27912285923957825, 0.05238794535398483, -0.26456597447395325, -0.24669629335403442, -2.4018971920013428, -0.4681406319141388, 0.08452047407627106, 0.08269979059696198, 2.1780002117156982, -0.8615289330482483, 1.2091648578643799, -1.5020668506622314, 0.2054530531167984, 0.265485942363739, -1.670233130455017, 0.8180579543113708, 0.13794204592704773, -1.5217748880386353, 0.7005295157432556, 0.74493807554245, -1.0249255895614624, -0.9329513311386108, 0.04032332822680473, 0.11399877071380615, 0.8377607464790344, 1.744200348854065, 0.3841550350189209, 0.49754437804222107, 0.20214375853538513, -1.2068496942520142, -1.8586148023605347, -1.1507893800735474, -0.18699099123477936, 0.042376551777124405, 0.23902356624603271, -0.11664027720689774, -0.9376332759857178, 2.020494222640991, -0.827431321144104, -0.5126696825027466, 0.9938586354255676, 0.21186795830726624, 1.82533860206604, -0.7899601459503174, 1.090681552886963, -0.82804936170578, 0.7504163980484009, -2.2515602111816406, -3.1728641986846924, -0.8267844915390015, -1.056792140007019, 1.7208092212677002, 1.67609441280365, -0.46144139766693115, -0.10148747265338898, 0.1784919649362564, 1.5471242666244507, 1.1460950374603271, -1.1160333156585693, -1.4288275241851807, -0.05485738441348076, 0.1112188994884491, 0.08879726380109787, 0.5511515140533447, 0.23459941148757935, 1.5363941192626953, 0.09780941158533096, 0.2859874963760376, -0.21105585992336273, -1.7749712467193604, -0.5049557685852051, -0.6618223786354065, -1.174530267715454, 0.17883560061454773, -4.457566738128662, -0.8948807716369629, 0.2373274564743042, -1.8165922164916992, 0.6686756610870361, -2.039884090423584, 0.4658748507499695, 1.003615379333496, 0.1071084588766098, -1.4484148025512695, -1.1270309686660767, -0.005616738460958004, -0.6450018882751465, 0.2603050470352173, 0.5605751276016235, -2.071941614151001, 0.7913297414779663, -0.1653304547071457, -0.5475406646728516, 0.5133751630783081, -0.0060415733605623245, 1.049721360206604, 1.7217893600463867, -0.8679565191268921, -0.6000723242759705, -0.4449816644191742, 0.5022903084754944, 2.5086193084716797, 1.972118854522705, 0.17153501510620117, 1.4161996841430664, -0.3972255289554596, -0.8981713056564331, 0.7120077610015869, 1.3480225801467896, -0.685077965259552, -0.765416145324707, 0.09515594691038132, -0.22370487451553345, -0.23999391496181488, -1.1777703762054443, 0.49369752407073975, 0.8586170673370361, -0.6800713539123535, -0.9754433631896973, -0.05959494039416313, -1.949081301689148, 0.08617878705263138, 1.7835670709609985, 0.9789372682571411, 0.38572895526885986, -0.24068498611450195, -0.30869606137275696, -0.4982219636440277, 1.5105055570602417, -1.1718792915344238, 0.17951563000679016, 0.034572962671518326, 1.4952287673950195, 0.2782299518585205, -0.5156323909759521, 1.5538368225097656, 0.5080007314682007, 0.3750784993171692, 0.23767851293087006, -0.43432363867759705, -1.8107049465179443, -0.906164824962616, -2.5102427005767822, 0.6660612225532532, -0.018752114847302437, -0.2537131607532501, -0.12484750896692276, -0.9649115800857544, -1.2238136529922485, -2.362678289413452, 0.15312343835830688, -0.3720032274723053, -1.4385368824005127, 0.6778305172920227, 0.5817276835441589, -0.2700928747653961, -1.5612537860870361, 0.2643999457359314, 0.7622978091239929, -1.0321203470230103, -0.7217154502868652, 0.07423818111419678, -0.37050631642341614, 1.381849765777588, -0.7372907996177673, 0.5570353865623474, 0.02852592244744301, 0.3424142897129059, 1.1344627141952515, 0.05431978031992912, 0.5235039591789246, 0.3099060654640198, -1.0097086429595947, -0.3357637822628021, 1.4711698293685913, -1.4220929145812988, 1.028111457824707, 0.1323002278804779, 1.428215503692627, 0.35872283577919006, -1.3429553508758545, 2.8003740310668945, -0.4246945083141327, 1.6217494010925293, -0.23803111910820007, 0.49407950043678284, -0.21928684413433075, -0.31773340702056885, -0.961768627166748, -0.6558251976966858, -1.461558222770691, -1.4872010946273804, -0.46377846598625183, 1.385448932647705, 1.0814801454544067, -1.0234670639038086, -0.052142415195703506, -0.10095648467540741, -0.9944629669189453, -0.8311887383460999, -0.9399095177650452, 0.5206970572471619, 1.0828913450241089, -0.43809548020362854, -0.5248620510101318, 0.3522890508174896, 0.49817565083503723, 0.33896172046661377, 1.0825163125991821, -0.7717171311378479, -0.7836256623268127, 0.33980777859687805, 0.2958395183086395, -1.0695353746414185, 1.151013731956482, -0.9347832798957825, 0.46042001247406006, 1.690434217453003, 0.43090078234672546, -0.41539373993873596, 0.422008752822876, 1.9357669353485107, -0.9350184202194214, 0.7176898121833801, -0.4199744760990143, -0.5356029868125916, 0.9259853363037109, 1.3975580930709839, -0.561874508857727, -0.7515225410461426, -0.15376926958560944, 0.360312283039093, 0.30902382731437683, 0.7600611448287964, 0.9283265471458435, -1.238614559173584, -1.079662799835205, 1.3941154479980469, -0.7061527967453003, 1.5638618469238281, 0.22881916165351868, -0.7415056228637695, -0.8670151829719543, 1.0745561122894287, -0.10613258183002472, 0.3126538395881653, -0.5271252393722534, -0.8159354329109192, 0.9715724587440491, 3.022894859313965, -0.6257017850875854, -1.4018235206604004, -0.4928715229034424, 0.5227369070053101, -0.6864367723464966, 0.26110410690307617, -0.6610075831413269, -0.4827063977718353, -0.5670705437660217, 0.12786227464675903, -0.4033352732658386, -0.014195534400641918, 0.7318958640098572, -0.8837756514549255, 0.335088312625885, -0.4015730023384094, -0.5512160658836365, 0.8675637245178223, 1.7478210926055908, -1.6289983987808228, -0.031029313802719116, -0.08693260699510574, -0.6648293137550354, 1.085505723953247, 1.8685319423675537, 0.13430805504322052, 0.16212666034698486, 1.0412505865097046, 2.063230276107788, 0.14433658123016357, -0.9356182813644409, -0.17979836463928223, 1.211334228515625, -0.6763920187950134, 0.9910991787910461, 0.9296272397041321, 0.544746458530426, 1.7293883562088013, -1.291458010673523, 0.40154343843460083, 0.24075910449028015, 0.5399978756904602, 0.7413308024406433, -0.6135480999946594, 0.622657835483551, 0.6625633239746094, -0.2770291566848755, -0.3659569025039673, 1.1876074075698853, -1.042763352394104, -0.0581778883934021, -0.7441044449806213, -0.06833546608686447, 0.7934845089912415, 0.046566687524318695, 1.0900377035140991, 1.186665415763855, 0.13894762098789215, -0.757627546787262, -1.0647530555725098, 0.2752975821495056, -1.9301657676696777, 0.9989904165267944, -0.4447237551212311, -0.30376115441322327, -0.8902867436408997, 0.40684008598327637, -0.7700813412666321, -1.2429423332214355, 1.3133354187011719, -0.5872587561607361, 0.753203809261322, 2.176410436630249, -0.030861366540193558, 0.07716866582632065, -0.1843918263912201, -0.5937590599060059, -0.6661641597747803, 0.5216764807701111, 1.0421867370605469, -2.647193193435669, -0.6383712291717529, 0.4249286949634552, 1.3580759763717651, -0.27410125732421875, -1.9931273460388184, 1.1719053983688354, 0.6513944268226624, -0.16201749444007874, -1.246151328086853, -0.5680778622627258, -1.2148979902267456, -0.541898787021637, 0.4396887421607971, -0.10975711792707443, -1.1916698217391968, -1.3353394269943237, 0.6570896506309509, -0.7269361019134521, 0.5265325903892517, 1.0401440858840942, -1.1740291118621826, -0.18852144479751587, -2.1649487018585205, -0.92581707239151, 1.3179928064346313, -0.7427611351013184, 0.22017095983028412, -0.04821622744202614, -0.3776293396949768, -0.10401985794305801, 0.15264374017715454, 1.6443575620651245, 0.2822761833667755, -1.0416834354400635, -0.9390148520469666, -0.08329545706510544, 0.2620224058628082, 0.8654457330703735, 0.49239078164100647, 0.05123338848352432, -0.30930617451667786, 1.1462124586105347, -1.0749704837799072, 0.3025207817554474, 0.2281540483236313, 0.6222400069236755, 0.23403944075107574, 1.5250285863876343, 1.2843257188796997, -0.07128342241048813, 0.6481245756149292, 0.6141853928565979, -0.8643965721130371, -0.35164064168930054, 0.18162991106510162, 0.6644949316978455, 0.04451940208673477, -0.3815610706806183, 0.4431920647621155, -0.4638011157512665, -1.0047597885131836, 0.7905243039131165, -1.0029096603393555, 0.6319471001625061, 0.6147403120994568, -0.5849830508232117, 0.13446347415447235, 0.35634350776672363, 0.6638368368148804, -1.1965800523757935, 0.009539431892335415, 0.7635654211044312, -1.1447070837020874, 0.7412958145141602, 0.2407427877187729, 0.5363761186599731, -0.43703243136405945, 0.38179299235343933, 0.06574998795986176, -0.23640991747379303, -0.9433237314224243, -0.00395973352715373, -0.42483603954315186, -1.5191529989242554, 0.6119988560676575, -2.176861524581909, -0.33598947525024414, 1.2903441190719604, -0.6263184547424316, -0.020747322589159012, -1.0471813678741455, -0.520568311214447, 1.2278695106506348, -0.9406999349594116, 0.2812979221343994, 0.506233811378479, 0.14881998300552368, -1.8384580612182617, 1.0624110698699951, 0.9346901774406433, -1.2227609157562256, 1.3354755640029907, -0.1817251294851303, 1.2543001174926758, 0.12130647897720337, 1.1710442304611206, 0.5048021078109741, 1.0646113157272339, 0.1395268440246582, 0.093437060713768, 0.779522180557251, -0.33073562383651733, -1.6388421058654785, 0.7783952951431274, 0.08967476338148117, -0.7907739281654358, 0.10470274090766907, 0.3654521107673645, 0.6678137183189392, 0.06586921960115433, -0.015291822142899036, 0.14670220017433167, 0.4366490840911865, 1.6379039287567139, -2.1285743713378906, 0.2222396582365036, -3.027754545211792, -0.8510528206825256, -0.7663018107414246, -1.5252883434295654, 0.695531964302063, -0.1620054990053177, 1.1026865243911743, -1.5951323509216309, -0.23754438757896423, -0.4346143305301666, -0.45132339000701904, 0.4753127098083496, -0.45695796608924866, 0.7340685725212097, -0.05498214066028595, 1.0508087873458862, 1.214011311531067, 0.10072694718837738, -0.8448102474212646, -0.2524474859237671, 0.2500850260257721, -0.6463265419006348, 0.8194342851638794, -0.0559530183672905, -0.9215158224105835, -0.16152869164943695, 0.41564062237739563, -0.9644915461540222, 1.1808834075927734, -1.3022693395614624, -1.0890703201293945, -0.6124404072761536, -2.5164809226989746, -0.8795690536499023, 1.5174647569656372, 0.47543612122535706, 0.11033431440591812, -1.384039044380188, 1.9522361755371094, -0.9850338697433472, -0.6458557844161987, 1.0780253410339355, 0.6351420879364014, -0.8372008800506592, -1.6181416511535645, -0.3859560191631317, 0.38152065873146057, -0.7842659950256348, 0.23026368021965027, 1.4181524515151978, -1.2401186227798462, -0.9301753044128418, -0.45448264479637146, -0.5047144293785095, -0.8964478969573975, 0.4831327199935913, -0.36534470319747925, 1.7195543050765991, 0.3214561939239502, 0.8872025012969971, -0.4558106064796448, 0.14452868700027466, 0.44717440009117126, -1.0136330127716064, 0.09486871212720871, 0.44952794909477234, -0.03553387150168419, -0.7694580554962158, -0.8320578336715698, 0.590438961982727, -0.1449662297964096, -1.746093988418579, -0.8800817131996155, 1.0429381132125854, -0.2314753383398056, -0.14260399341583252, 1.2600747346878052, -0.8514260649681091, 0.8772961497306824, 0.6299750804901123, 0.1861352175474167, -0.9679204225540161, 0.49999940395355225, 0.8507924675941467, -1.055084228515625, -1.8823868036270142, -1.677620768547058, 1.251132845878601, 0.5210652947425842, 1.2227224111557007, 0.8148274421691895, 0.7472647428512573, 1.3457099199295044, 2.061347484588623, 1.3498854637145996, -0.589399516582489, 0.3367023169994354, 1.4396392107009888, -1.3730144500732422, 1.665908694267273, 0.3982606530189514, 0.45785585045814514, -0.20328617095947266, -0.16079792380332947, -0.22008457779884338, 0.5048694014549255, -1.0571491718292236, 1.0135442018508911, -0.6622507572174072, 0.9294661283493042, -0.7538756132125854, -1.3255046606063843, -2.0941691398620605, 0.9010744094848633, -1.4529715776443481, -1.1284621953964233, -1.4433289766311646, -1.3688164949417114, 0.1257689893245697, 0.7120858430862427, 0.35560688376426697, 1.237966775894165, -0.02284983918070793, 0.5132761001586914, 0.5098252296447754, -0.5646805167198181, -0.6470768451690674, -0.24714304506778717, -0.3853056728839874, -1.7399952411651611, 1.3676161766052246, 1.124112606048584, -0.6838781833648682, 1.0957167148590088, 1.0182766914367676, 0.02488255500793457, 1.4504516124725342, -0.5423439145088196, 2.036702871322632, -1.738959789276123, -0.16458627581596375, -1.9764297008514404, -0.8518995642662048, -0.06116500869393349, 0.02322559989988804, 0.3360511362552643, 0.24734264612197876, 0.8549847602844238, -0.06387226283550262, -1.2234290838241577, 0.06785429269075394, 1.7007579803466797, -0.9708634614944458, -0.20221689343452454, 1.114677906036377, 0.22253374755382538, -1.0011944770812988, -0.8380007743835449, -0.9544593691825867, -0.9801231622695923, 0.2756634056568146, 2.198122262954712, 0.26155945658683777, 0.29867982864379883, 0.6325051188468933, 1.0811994075775146, 0.4640844762325287, 0.14395233988761902, 0.9048912525177002, -0.23460231721401215, 1.04203462600708, -1.7303160429000854, 0.664290189743042, 0.619341254234314, 0.44234544038772583, -2.4205658435821533, -0.401349812746048, -0.9001826643943787, 0.08940635621547699, 0.17697052657604218, -0.07672446221113205, 1.806304693222046, -0.46621593832969666, -1.7007075548171997, 1.9577611684799194, 1.9118603467941284, -0.630509614944458, 0.2721521258354187, 0.782900333404541, 2.038896083831787, -0.5766811966896057,-0.3859558701515198, 0.7933176159858704, 1.3765407800674438, -0.34779104590415955, -0.9430199265480042, -0.4077472686767578, -2.603008985519409, 0.3115611970424652, 1.0416357517242432, 1.547062873840332, 0.5472713708877563, -0.512486457824707, 0.7974473237991333, 0.28667718172073364, 0.1354890763759613, 0.6234353184700012, -0.07820867002010345, -1.3802556991577148, 0.8748401999473572, 1.3638468980789185, 0.5694319605827332, -0.898334801197052, 1.5010206699371338, -0.523626983165741, 0.11108909547328949, -0.6651378273963928, 0.08493851870298386, 0.4033900797367096, 1.8130345344543457, -0.7316403985023499, -0.5542997717857361, -0.7731867432594299, 0.43395230174064636, 0.9147107601165771, -0.4608747661113739, 0.9459581971168518, 0.4335741400718689, 0.6825537085533142, 0.995204746723175, 1.2014477252960205, -0.8399257659912109, 1.7603645324707031, 1.0264159440994263, -0.4712759256362915, -0.9875678420066833, 0.3992365896701813, -1.2146669626235962, 0.357363224029541, -2.177626132965088, 0.9398990273475647, -0.1301313042640686, -0.4474254250526428, -0.8689817786216736, -0.38743504881858826, -0.14580348134040833, -0.3906286060810089, 1.286295771598816, -1.7776163816452026, 2.076753854751587, 0.6974522471427917, -0.23099204897880554, 0.11509230732917786, -1.1259214878082275, 0.37161147594451904, 1.947410225868225, 0.056228138506412506, -0.3280990421772003, -0.6872709393501282, 0.7018030881881714, -0.4944825768470764, -0.1019381731748581, -1.1134710311889648, -1.687645435333252, 1.0320042371749878, -0.792866587638855, -1.9030263423919678, 1.9030191898345947, 0.47158950567245483, 0.9945462942123413, -1.012483835220337, 0.27064114809036255, 0.6676345467567444, 0.9805602431297302, -0.2154299020767212, -0.059692542999982834, -0.255761981010437, 0.05143682658672333, -0.5356511473655701, 0.9172849655151367, -1.3105610609054565, -0.5079241394996643, 0.5912820100784302, 0.1397896111011505, -0.4712033271789551, 1.4596624374389648, -0.102261483669281, -0.2612711489200592, -1.031201958656311, -0.26775169372558594, 2.711669445037842, -0.5636120438575745, -0.7866376638412476, -0.1052582859992981, -0.7292200326919556, 0.5680748224258423, 1.8549467325210571, -0.5459160804748535, -0.29830676317214966, -0.22569113969802856, -0.9278537631034851, -1.4082926511764526, -0.38115009665489197, 0.24087898433208466, -0.6152486205101013, 0.15638327598571777, 0.28101375699043274, -0.421351820230484, -0.2939402163028717, -0.8992107510566711, 0.17619222402572632, -0.48678162693977356, 0.4456979036331177, -0.143698588013649, 0.23693296313285828, -0.1346772313117981, 0.051839813590049744, -1.1595426797866821, -0.7403738498687744, 0.665787935256958, 0.022081129252910614, 1.0720897912979126, -0.8827539682388306, 0.21281345188617706, 0.46751609444618225, -1.5226353406906128, -0.09747330844402313, -0.03313560038805008, -0.21578721702098846, 0.5406012535095215, 0.5115357041358948, 0.34015336632728577, 1.0359445810317993, -0.18953225016593933, 0.5561923980712891, 0.23158997297286987, 0.35251694917678833, 1.9253175258636475, -0.24850969016551971, -0.7252833247184753, 0.7390220761299133, 1.1051920652389526, -0.9112038016319275, 0.38003969192504883, -0.28567051887512207, 1.2382957935333252, -0.014393108896911144, -0.019318697974085808, 3.069650888442993, -0.3749072551727295, -0.6480314135551453, -1.926870346069336, -0.669609785079956, -0.9168500304222107, 0.2952199876308441, -2.117751121520996, 0.5227321982383728, 2.0267114639282227, -0.6014419198036194, -1.175646185874939, 0.9997896552085876, 0.7248967885971069, 0.6786755323410034, 0.022491931915283203, 0.9935764670372009, 1.5197066068649292, 0.26662102341651917, -1.144931674003601, -0.7569316625595093, -1.360493540763855, -1.1387805938720703, -0.4178517758846283, 1.5027414560317993, -0.3510243892669678, 0.6666653156280518, 1.0525099039077759, 0.4671882688999176, 0.6286123991012573, 0.9309514164924622, -0.2349110245704651, -1.186299443244934, 1.7340900897979736, -0.8929686546325684, 0.8960447311401367, -0.5849615335464478, -0.91832435131073, -1.6720428466796875, -0.11677008122205734, 0.0681489035487175, 1.1627037525177002, 0.14044736325740814, 0.7459562420845032, 0.21189948916435242, 0.4644404947757721, -0.3586905002593994, 0.8919798731803894, -0.2639780044555664, 0.41645383834838867, 0.6021699905395508, 0.3641333281993866, -0.794434130191803, -0.4622030556201935, 0.7890366911888123, -0.7385450005531311, -0.10492058098316193, 0.8550023436546326, 1.3582630157470703, -1.4266165494918823, 0.3737516701221466, 1.4050939083099365, -0.12538914382457733, -0.5616176128387451, -0.3387034833431244, 0.42131584882736206, -1.2214511632919312, 1.3163740634918213, 0.6297095417976379, -0.9198814034461975, -0.16059556603431702, 0.09301616251468658, -0.7748075127601624, -2.3035812377929688, -0.19608332216739655, 0.6168276071548462, -0.2803472578525543, 1.6754662990570068, 1.7687841653823853, 0.9705377221107483, 0.210821732878685, -1.833869218826294, -1.0119407176971436, -0.23213601112365723, -0.5438799262046814, -1.2747596502304077, 0.45459240674972534, -0.5337063670158386, -1.8801586627960205, -0.014536894857883453, 0.7514694333076477, 0.3202367424964905, 2.7560057640075684, -0.24967019259929657, 1.4984830617904663, 1.3615010976791382, 0.42033347487449646, -0.06551782041788101, -0.9475528001785278, -0.7021328806877136, 0.05702374503016472, -0.7720993161201477, -1.107029914855957, -1.2851717472076416, -1.4287278652191162, 0.16599415242671967, 0.6349439024925232, -2.1458656787872314, -1.3646799325942993, -0.04705968126654625, -0.2857394814491272, -0.18842697143554688, 0.062252167612314224, -0.042301520705223083, 0.6800511479377747, -0.010988223366439342, -1.270578384399414, -1.0624690055847168, -1.4183471202850342, 1.3655434846878052, 0.11190415918827057, 1.4819689989089966, -0.10744388401508331, -0.8064826130867004, 0.8721147775650024, -0.2599572539329529, -1.6440045833587646, 0.7031749486923218, -0.8190003633499146, -0.023605521768331528, -0.7509631514549255, -0.7638696432113647, 0.1094333827495575, 1.017399549484253, 0.04625942185521126, 2.1267642974853516, -1.3583834171295166, -0.7492795586585999, -1.3572252988815308, -0.06151806190609932, -0.27185341715812683, 0.5841495394706726, -0.036132507026195526, 0.04765389859676361, -0.25569096207618713, -0.7652139067649841, 1.078226923942566, 0.5305820107460022, -1.887202262878418, -1.506195306777954, -0.5097988247871399, -2.3877336978912354, 0.6056820750236511, 0.570992112159729, -0.3883848488330841, -0.9489464163780212, -0.5769287943840027, -0.3390989899635315, 0.013106055557727814, 2.0570626258850098, -1.1291505098342896, 0.20591993629932404, -0.7067580223083496, -1.194834589958191, -1.210514783859253, 1.3492364883422852, 0.2199171483516693, 0.16145923733711243, 0.6398746371269226, -0.11919469386339188, 2.0263750553131104, 0.36786341667175293, -1.9530433416366577, 0.45082324743270874, 0.28519535064697266, -0.7815859913825989, -0.5998251438140869, -0.0730707198381424, 1.559450387954712, -0.7059773206710815, 0.05711636319756508, 1.8524550199508667, -0.5993990898132324, 0.5407602190971375, 0.25027117133140564, -1.311682939529419, -0.5616074800491333, -0.14707061648368835, -0.28102537989616394, 2.991621732711792, 0.2595294713973999, 0.12634679675102234, 0.3375697433948517, -2.491361141204834, 0.8580591082572937, -0.1096847802400589, 1.0264201164245605, 0.6745820641517639, 0.5594793558120728, -0.2562224864959717, 0.5694603323936462, -1.0544339418411255, -0.47052472829818726, 0.8249353170394897, -1.7505422830581665, 0.8350671529769897, 0.3287014365196228, -0.8424237966537476,-0.7031763195991516, 2.069502353668213, -1.048577904701233, -0.4686150848865509, -1.1126518249511719, -1.2684106826782227, -0.3934065103530884, 0.3959363102912903, -1.4345157146453857, 1.6905033588409424, 0.7103226184844971, 0.00706832017749548, -0.3729831278324127, 0.7175149917602539, 0.4520626664161682, -0.45103007555007935, 0.2866237759590149, -0.8329823017120361, -0.7741301655769348, 0.030885493382811546, -1.248436450958252, -1.9199049472808838, -0.5593400597572327, -1.9907909631729126, -1.4970990419387817, -0.9631341695785522, -3.125248908996582, -0.3542078733444214, 1.3598521947860718, 1.080370306968689, 0.03440188243985176, -0.7168768644332886, -0.32592353224754333, 0.9930990934371948, 0.7213590741157532, -1.816176176071167, -0.18912218511104584, 1.6842303276062012, -1.0512933731079102, -0.4270818829536438, 1.9639877080917358, 1.7484346628189087, -1.0876226425170898, -1.419143557548523, -0.9659984111785889, -1.5649954080581665, -1.1707053184509277, 0.8161000609397888, -1.102698802947998, -0.2558152377605438, -0.5450940132141113, 0.03297295793890953, -0.6470919251441956, -1.8265681266784668, 0.9044525027275085, -0.9888673424720764, 0.4936091899871826, -0.35192257165908813, -0.5499001741409302, -0.18267792463302612, 1.1800005435943604, 0.1676735132932663, 0.5480664372444153, 0.10650783777236938, -0.6907669305801392, -0.4953933358192444, -0.46742916107177734, 0.05571671202778816, -0.5182346105575562, 0.5335946083068848, 0.20580586791038513, 0.10447815805673599, -0.5980430841445923, -0.5264541506767273, -1.704163908958435, -0.1021147221326828, -1.1743714809417725, 1.5016355514526367, -0.9092674851417542, -0.4403754472732544, 0.10361018031835556, -0.2890578806400299, -0.9674746990203857, -0.5342630743980408, 0.5884268283843994, -0.24277272820472717, 1.02679443359375, -0.9577941298484802, -0.5751011967658997, -1.2010560035705566, 0.016083234921097755, -0.8017513751983643, 0.4038708209991455, 0.6794184446334839, 0.015398974530398846, -0.28537890315055847, 0.33527469635009766, -0.9029396772384644, 0.4081169366836548, -1.5749337673187256, -0.8864859342575073, 0.5222132802009583, 1.4491773843765259, 0.2935207784175873, -0.7156323194503784, 0.9484111070632935, -0.4770437180995941, 0.7580826282501221, -1.150391697883606, 1.0240907669067383, 1.7751405239105225, -1.2695072889328003, -0.9774441123008728, -0.39846742153167725, 1.3685623407363892, -0.2946776747703552, -1.2782976627349854, -0.06557011604309082, -0.11361865699291229, -1.541324496269226, 0.13307824730873108, 1.674526572227478, 0.7330602407455444, -0.25523558259010315, 0.22356624901294708, 0.17123933136463165, -0.1486121118068695, 0.64533531665802, 1.0686872005462646, -0.21541717648506165, -0.32832884788513184, 1.4761207103729248, 1.7750470638275146, -0.2349761575460434, 1.5020309686660767, 1.8342485427856445, 0.691299557685852, 0.6542538404464722, -0.21254776418209076, -0.2604779601097107, 0.08008672297000885, 0.19567447900772095, 2.0589189529418945, -0.6017441153526306, -1.4072359800338745, -0.23829516768455505, -1.4842233657836914, 0.22129254043102264, 0.42094579339027405, 1.6161563396453857, -0.1208776906132698, -0.5427080988883972, 0.4268173277378082, -0.1680992841720581, 0.6079670786857605, -0.7995480895042419, -0.310797780752182, 0.5615962743759155, -0.49516573548316956, -0.07269269973039627, 0.09172696620225906, 2.941441535949707, 0.3629891574382782, 1.0415844917297363, 0.3124113976955414, 1.0769879817962646, -0.5234696269035339, -0.779076337814331, -0.5376963019371033, -0.4013535678386688, 0.9472349882125854, 0.09447301924228668, 2.292386293411255, -1.1162891387939453, -1.5334347486495972, 0.7863227725028992, -0.3838779926300049, 0.0038755363784730434, 0.01860208809375763, 0.4943521320819855, 0.906863272190094, -1.7200202941894531, 0.9100423455238342, -0.35374146699905396, 0.39257848262786865, -0.6462534666061401, 0.4055567681789398, 1.1336747407913208, -0.1672714650630951, -0.8839408755302429, 1.3521075248718262, -0.6813105940818787, -0.7928037047386169, 0.16517113149166107, 0.3839418292045593, 1.0600895881652832, 0.8313129544258118, 0.8032035827636719, -1.2921428680419922, 1.828239917755127, 0.7824868559837341, 0.47728991508483887, 0.2283540964126587, -0.11331984400749207, -0.2869618833065033, -1.3936010599136353, 0.18490062654018402, -0.6959137916564941, -2.0707995891571045, -0.8657941818237305, -0.2643855810165405, -1.1052582263946533, -1.911431908607483, 1.0332310199737549, 0.17980390787124634, 0.1407242864370346, -1.3253751993179321, -1.266922116279602, 1.3367278575897217, 0.5178878307342529, -0.16030246019363403, 0.6081222891807556, -0.621526300907135, 1.6016674041748047, -0.5686457753181458, 0.057816680520772934, -0.5481541752815247, 1.1421394348144531, -0.1107446551322937, 0.6791332960128784, 0.0018254707101732492, -1.7024898529052734, -0.07082581520080566, 0.022739198058843613, 0.507290780544281, 0.23266251385211945, 0.3731221556663513, -0.06156414374709129, -0.10427962243556976, 0.424711138010025, -1.9804619550704956, 0.35693082213401794, -0.23635348677635193, 0.8622715473175049, -0.23959632217884064, -0.023887934163212776, 0.16107195615768433, 0.029796909540891647, 0.5003473162651062, 0.24421215057373047, 1.872403621673584, -0.3300632834434509, 1.0847852230072021, -0.4882701635360718, -3.0069210529327393, 1.3164578676223755, 1.8596160411834717, -0.47270888090133667, 0.8211718201637268, -0.12048285454511642, 1.3187865018844604, 0.14681574702262878, -0.2653231620788574, 1.8469749689102173, 0.8525374531745911, 0.5859884023666382, 0.11466021835803986, -0.15283308923244476, 1.0659066438674927, -0.25631383061408997, 0.2644650340080261, 0.31048139929771423, 1.2255010604858398, 0.7625424265861511, 0.07109476625919342, -1.9515167474746704, 0.7415394186973572, -0.08381898701190948, 0.7361668944358826, 0.5222479701042175, -0.7718835473060608, 0.3788865804672241, 0.13842423260211945, 0.2033444195985794, 1.106125831604004, 0.16991537809371948, -0.5564043521881104, 1.378411054611206, -1.6352988481521606, -1.7723474502563477, 0.08739284425973892, -1.1349016427993774, -0.4408138692378998, 1.071467399597168, -0.14765867590904236, -0.5783859491348267, -0.4474959969520569, 1.0266544818878174, -1.0136781930923462, -0.37455371022224426, -1.2476717233657837, 1.3244708776474, 0.4088573753833771, 0.7085264921188354, 0.5176963210105896, -1.1482266187667847, 0.4562559425830841, -0.05348188430070877, -0.9142765402793884, 0.5124844908714294, -0.034451067447662354, -0.2541334629058838, -0.06810668855905533, -0.782544732093811, -0.16091659665107727, 0.7708766460418701, 0.4143134653568268, 0.3831807076931, 0.22664014995098114, 0.43566378951072693, 0.7430370450019836, -0.8917005062103271, -1.5206760168075562, -0.9015985727310181, -1.8927280902862549, 1.3108515739440918, 0.7065114974975586, 0.8898401856422424, 1.0611813068389893, 1.359848976135254, -0.8050362467765808, -3.2852511405944824, 1.5680139064788818, -1.0311942100524902, -0.9306149482727051, 1.9667820930480957, -0.3321269154548645, 0.08320575207471848, -0.2601674497127533, -0.6249432563781738, 0.17216990888118744, 1.5597569942474365, 0.2873627543449402, -0.6242672801017761, -1.3757975101470947, 0.01483726967126131, 0.2248803824186325, 0.4624071717262268, -0.9925063848495483, -0.493473082780838, 0.7004188299179077, 1.6205023527145386, 1.4291768074035645, -0.11880000680685043, -0.5403986573219299, 0.6990835070610046, 0.3374771773815155, 0.507012128829956, -0.015236261300742626, 0.20656095445156097, 0.2450937032699585, -1.3595020771026611, 1.9418766498565674, 0.9274846315383911, -1.0056668519973755, 0.9003636240959167, -0.20641422271728516, -2.541029214859009, 0.4666680097579956, 0.5404683351516724, -0.21295198798179626, 0.5497145652770996, 0.7899172306060791, -0.2317177653312683, -0.07422992587089539, 0.5211905837059021, -0.04005570337176323, 0.5272336602210999, 0.75503009557724, 1.22007155418396, 0.15686407685279846, 1.5773200988769531, -0.33874011039733887, -0.285367488861084, 1.2392011880874634, 0.16648122668266296, -0.19662731885910034, 0.5416107177734375, 0.284048855304718, 0.7807462215423584, 0.0772693082690239, 0.7168674468994141, -0.4005580544471741, 0.2821873426437378, -0.37519845366477966, 0.0027425482403486967, 0.9650689363479614, 0.6988857388496399, -0.7583848237991333, 0.4480089545249939, -1.2746529579162598, -0.045221660286188126, -0.6292626857757568, 0.051249124109745026, 0.06223708763718605, 1.7673089504241943, 1.4924697875976562, 0.17977911233901978, 1.6981745958328247, 0.1672186404466629, -1.1172816753387451, 0.9083341360092163, -0.67180335521698, 0.5286462903022766, 1.2448959350585938, -1.2001924514770508, 1.1921318769454956, -0.3351643681526184, 1.206398844718933, 0.24663867056369781, -0.2858778238296509, -0.2676036059856415, -0.9091373682022095, -0.16211582720279694, 0.5029604434967041, 1.1972564458847046, 1.4170546531677246, -0.4289688766002655, -0.18716362118721008, -1.1177399158477783, 0.10566428303718567, -1.0103216171264648, -1.790848731994629, 0.1317184865474701, -0.7092040777206421, -0.8601858615875244, 1.3682377338409424, -0.1335560530424118, 1.3761720657348633, -0.1975099891424179, 0.079250268638134, -0.0867479220032692, 1.072158932685852, 0.5893349647521973, 0.6763414740562439, 1.0571171045303345, -1.014441728591919, -1.9205706119537354, -1.0072028636932373, -1.0784144401550293, -0.996205747127533, -0.4991379380226135, -0.45854923129081726, 1.7497930526733398, 0.6659351587295532, -0.07168318331241608, -0.6838850378990173, 0.18619421124458313, -0.3036673963069916, 1.3667840957641602, -1.6135997772216797, 1.0031803846359253, 0.18819868564605713, 1.265553593635559, -0.7307321429252625, 0.46138429641723633, 0.550912082195282, 1.4510517120361328, -0.5674338936805725, -1.1451467275619507, -0.765305757522583, 0.04635929688811302, -0.4515441358089447, -0.15090897679328918, 1.5848568677902222, 0.9725602865219116, -1.2359994649887085, -0.6369375586509705, 0.7201210856437683, -0.97325199842453, 0.5723949670791626, -1.6268988847732544, -0.6424503922462463, 1.1688640117645264, 0.16156013309955597, 0.4514414668083191, -0.6781786680221558, -1.3265111446380615, -0.05630654841661453, -0.9594501852989197, -0.3388804793357849, -0.21194304525852203, -0.534602165222168, -2.895601749420166, 0.3603569567203522, 0.017375523224473, 0.14073063433170319, -1.5897867679595947, -0.5102773904800415, 0.47190824151039124, 0.0754183977842331, -1.6053121089935303, 0.14352358877658844, -1.3024070262908936, -1.0437709093093872, -1.4557719230651855, 0.895388662815094, -0.4345977008342743, -1.272947907447815, 1.1944854259490967, -0.8900570273399353, -0.5897367000579834, 0.3933540880680084, 0.3240635097026825, 0.5480417609214783, -0.5341367721557617, 0.6367537975311279, 1.9256292581558228, -1.75221848487854, -0.41230466961860657, -0.5778443813323975, -0.6978498101234436, 0.5793729424476624, 0.571374773979187, -0.42625531554222107, -1.029433250427246, 1.2258691787719727, 0.5204987525939941, 1.9958304166793823, 0.09828633815050125, 0.7816393375396729, -0.9274090528488159, -0.11356530338525772, -1.2144962549209595, -0.9480794072151184, 0.5247544050216675, 1.3974066972732544, -0.5880110859870911, 0.6322609782218933, -0.7389049530029297, -1.1852071285247803, 0.30875489115715027, 0.9301239848136902, -0.9930086135864258, -1.9672702550888062, 0.6127535700798035, -0.5060888528823853, 0.8578346967697144, 2.247459650039673, -0.7072847485542297, -0.9808002710342407, 0.2816307842731476, 0.6546081900596619, 0.2289007604122162, -0.24230267107486725, 0.3959594666957855, -1.1941598653793335, -0.7322438359260559, 1.2096837759017944, 1.2835453748703003, 1.7734615802764893, -0.3418593108654022, -0.0902809426188469, 0.3550195097923279, 2.027440071105957, -1.2974379062652588, 0.5419133305549622, 2.6244606971740723, 1.105350375175476, -1.6724731922149658, 0.1750856339931488, 0.21203479170799255, 0.37708204984664917, 0.8567672371864319, 1.1587073802947998, -1.2918639183044434, -0.4195218086242676, 0.32782790064811707, 0.4991025924682617, 0.812545895576477, 1.0040684938430786, 0.10502073168754578]; + let ground_truth = vec![-0.6489750146865845, -5.026195526123047, 5.606937408447266, 0.8131773471832275, -0.1522408127784729, -4.360945701599121, -1.3974180221557617, 11.217863082885742, -8.185169219970703]; let model_output = score_model.score_batch(&input).unwrap(); println!("{:?} {:?}", model_output, ground_truth); for (x, y) in model_output.iter().zip(ground_truth.iter()) {