From 3e0fd425794e3e0160ae6132916beff78823888a Mon Sep 17 00:00:00 2001 From: Andrej Karpathy Date: Fri, 13 Jan 2023 00:57:15 +0000 Subject: [PATCH] more scaling laws, clarification, and add simple interpolation of Approach 2 --- scaling_laws.ipynb | 189 ++++++++++++++++++++++++++++++--------------- 1 file changed, 128 insertions(+), 61 deletions(-) diff --git a/scaling_laws.ipynb b/scaling_laws.ipynb index c82deb6..e0e8fbd 100644 --- a/scaling_laws.ipynb +++ b/scaling_laws.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Trying to reproduce results from [Chinchilla](https://arxiv.org/pdf/2203.15556.pdf):" + "Reproducing some scaling laws results from [Chinchilla](https://arxiv.org/pdf/2203.15556.pdf). Can't get the numbers to match exactly, but can still be used as a rough guide to help determine compute-optimal models. Also contains related utilities for calculating flops and param counts." ] }, { @@ -136,51 +136,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "our estimated params: 43.7094M, chinchilla params: 44.0000M, d_model: 512, n_heads: 8, n_layers: 8\n", - "our estimated params: 57.3287M, chinchilla params: 57.0000M, d_model: 576, n_heads: 9, n_layers: 9\n", - "our estimated params: 73.8253M, chinchilla params: 74.0000M, d_model: 640, n_heads: 10, n_layers: 10\n", - "our estimated params: 89.8285M, chinchilla params: 90.0000M, d_model: 640, n_heads: 10, n_layers: 13\n", - "our estimated params: 105.8317M, chinchilla params: 106.0000M, d_model: 640, n_heads: 10, n_layers: 16\n", - "our estimated params: 116.7283M, chinchilla params: 117.0000M, d_model: 768, n_heads: 12, n_layers: 12\n", - "our estimated params: 139.7660M, chinchilla params: 140.0000M, d_model: 768, n_heads: 12, n_layers: 15\n", - "our estimated params: 162.8037M, chinchilla params: 163.0000M, d_model: 768, n_heads: 12, n_layers: 18\n", - "our estimated params: 174.9745M, chinchilla params: 175.0000M, d_model: 896, n_heads: 14, n_layers: 14\n", - "our estimated params: 195.8746M, chinchilla params: 196.0000M, d_model: 896, n_heads: 14, n_layers: 16\n", - "our estimated params: 216.7747M, chinchilla params: 217.0000M, d_model: 896, n_heads: 14, n_layers: 18\n", - "our estimated params: 251.1196M, chinchilla params: 251.0000M, d_model: 1024, n_heads: 16, n_layers: 16\n", - "our estimated params: 278.4133M, chinchilla params: 278.0000M, d_model: 1024, n_heads: 16, n_layers: 18\n", - "our estimated params: 305.7070M, chinchilla params: 306.0000M, d_model: 1024, n_heads: 16, n_layers: 20\n", - "our estimated params: 424.6938M, chinchilla params: 425.0000M, d_model: 1280, n_heads: 10, n_layers: 18\n", - "our estimated params: 488.6490M, chinchilla params: 489.0000M, d_model: 1280, n_heads: 10, n_layers: 21\n", - "our estimated params: 509.3356M, chinchilla params: 509.0000M, d_model: 1408, n_heads: 11, n_layers: 18\n", - "our estimated params: 552.6042M, chinchilla params: 552.0000M, d_model: 1280, n_heads: 10, n_layers: 24\n", - "our estimated params: 586.7150M, chinchilla params: 587.0000M, d_model: 1408, n_heads: 11, n_layers: 21\n", - "our estimated params: 632.3389M, chinchilla params: 632.0000M, d_model: 1536, n_heads: 12, n_layers: 19\n", - "our estimated params: 664.0945M, chinchilla params: 664.0000M, d_model: 1408, n_heads: 11, n_layers: 24\n", - "our estimated params: 724.4206M, chinchilla params: 724.0000M, d_model: 1536, n_heads: 12, n_layers: 22\n", - "our estimated params: 816.5023M, chinchilla params: 816.0000M, d_model: 1536, n_heads: 12, n_layers: 25\n", - "our estimated params: 892.8138M, chinchilla params: 893.0000M, d_model: 1792, n_heads: 14, n_layers: 20\n", - "our estimated params: 1018.1338M, chinchilla params: 1018.0000M, d_model: 1792, n_heads: 14, n_layers: 23\n", - "our estimated params: 1143.4537M, chinchilla params: 1143.0000M, d_model: 1792, n_heads: 14, n_layers: 26\n", - "our estimated params: 1265.7869M, chinchilla params: 1266.0000M, d_model: 2048, n_heads: 16, n_layers: 22\n", - "our estimated params: 1424.5576M, chinchilla params: 1424.0000M, d_model: 2176, n_heads: 17, n_layers: 22\n", - "our estimated params: 1429.4569M, chinchilla params: 1429.0000M, d_model: 2048, n_heads: 16, n_layers: 25\n", - "our estimated params: 1593.1269M, chinchilla params: 1593.0000M, d_model: 2048, n_heads: 16, n_layers: 28\n", - "our estimated params: 1609.3196M, chinchilla params: 1609.0000M, d_model: 2176, n_heads: 17, n_layers: 25\n", - "our estimated params: 1730.7878M, chinchilla params: 1731.0000M, d_model: 2304, n_heads: 18, n_layers: 24\n", - "our estimated params: 1794.0815M, chinchilla params: 1794.0000M, d_model: 2176, n_heads: 17, n_layers: 28\n", - "our estimated params: 2006.9637M, chinchilla params: 2007.0000M, d_model: 2304, n_heads: 18, n_layers: 28\n", - "our estimated params: 2283.1396M, chinchilla params: 2283.0000M, d_model: 2304, n_heads: 18, n_layers: 32\n", - "our estimated params: 2298.0403M, chinchilla params: 2298.0000M, d_model: 2560, n_heads: 20, n_layers: 26\n", - "our estimated params: 2638.9811M, chinchilla params: 2639.0000M, d_model: 2560, n_heads: 20, n_layers: 30\n", - "our estimated params: 2979.9219M, chinchilla params: 2980.0000M, d_model: 2560, n_heads: 20, n_layers: 34\n", - "our estimated params: 3468.9339M, chinchilla params: 3530.0000M, d_model: 2688, n_heads: 22, n_layers: 36\n", - "our estimated params: 3802.8109M, chinchilla params: 3802.0000M, d_model: 2816, n_heads: 22, n_layers: 36\n", - "our estimated params: 4152.0233M, chinchilla params: 4084.0000M, d_model: 2944, n_heads: 22, n_layers: 36\n", - "our estimated params: 4516.5711M, chinchilla params: 4516.0000M, d_model: 3072, n_heads: 24, n_layers: 36\n", - "our estimated params: 6796.2747M, chinchilla params: 6796.0000M, d_model: 3584, n_heads: 28, n_layers: 40\n", - "our estimated params: 9294.0206M, chinchilla params: 9293.0000M, d_model: 4096, n_heads: 32, n_layers: 42\n", - "our estimated params: 11714.6222M, chinchilla params: 11452.0000M, d_model: 4352, n_heads: 32, n_layers: 47\n", "our estimated params: 12296.1623M, chinchilla params: 12295.0000M, d_model: 4608, n_heads: 36, n_layers: 44\n", "our estimated params: 13124.4826M, chinchilla params: 12569.0000M, d_model: 4608, n_heads: 32, n_layers: 47\n", "our estimated params: 14614.4279M, chinchilla params: 13735.0000M, d_model: 4864, n_heads: 32, n_layers: 47\n", @@ -190,7 +145,7 @@ } ], "source": [ - "for m in chilchilla_models:\n", + "for m in chilchilla_models[-5:]: # only print last 5 models of the table\n", " p, d, f, k, h, l = m\n", " nparams = chinchilla_params(seq_len = 1024, vocab_size = 32000, d_model = d, num_heads = h, num_layers = l, ffw_size=f)\n", " print(f\"our estimated params: {nparams/1e6:.4f}M, chinchilla params: {p/1e6:.4f}M, d_model: {d}, n_heads: {h}, n_layers: {l}\")" @@ -201,7 +156,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We are almost able to reproduce the parameter counts for the Chinchilla models. TODO resolve...\n", + "We are almost able to reproduce the parameter counts for the Chinchilla models.\n", "\n", "Now turning to FLOPs:" ] @@ -472,7 +427,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## scaling laws" + "## Scaling Laws: Approach 3\n", + "\n", + "In their \"Aproach 3\", Chinchilla paper fits a function L(N,D) to approximate the final loss gives the model size and the data size. Here is the final fit:" ] }, { @@ -483,7 +440,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -555,7 +512,7 @@ "output_type": "stream", "text": [ "best model size: 316.23M\n", - "best dataset size: 10.12B\n" + "best dataset size: 11.65B\n" ] }, { @@ -570,7 +527,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEqCAYAAABEE9ZrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuW0lEQVR4nO3deViU9f7/8ecMy7Ajm4AspqIoKoqaCpor5m7W6ZfHLKy0Tp0sy7IT1Tl1Wg6ezJZz8tt6ytZjqannWC7pSS3FDTTRVMIFEFkUkWWQbeb+/QFOoiAIw9wzzPtxXXNdcnPPzHuY5tXn/sxn0SiKoiCEEDZGq3YBQgjREhJeQgibJOElhLBJEl5CCJsk4SWEsEkSXkIImyThJYSwSY5qF2BpRqORM2fO4OnpiUajUbscIcQVFEWhtLSUTp06odU23r6yu/A6c+YMYWFhapchhGhCdnY2oaGhjf7e7sLL09MTqP3DeHl5qVxNO6bXQ6dOtf8+cwbc3dWtR9iMkpISwsLCTJ/VxthdeF26VPTy8pLwaksODr/928tLwktct6a6daTDXghhkyS8hBA2ScJLCGGTJLyEEDZJwksIYZMkvIQQNknCSwjR5j7ecZK1B3Ioqag222NKeAkh2lS1wciSTenMX36ArMJysz2uhJcQok0dyL5AWWUNvu7ORAWbb2C4hJcQok39mH4WgLhufmi15lsMQcJLCNGmfsw4B8CI7gFmfVwJLyFEmykur+bn7AsADO/ub9bHlvASQrSZ5BPnMCrQLcCdTh1czfrYEl5CiDaz/dfaS8abzHzJCBJeQog29JMpvMx7yQgSXkKINpJZqCfrfDmOWg1DuvqZ/fElvIQQbeLHulbXgM4+eOjMv+6phJcQok2YLhkjzH/JCBJeQog2UGMwsuN4XXj1MH9nPUh4CSHawMGcYkoravB2daJviHebPIeElxDC7LYdq50SNCzCDwczTgm6nISXEMLsttbNZxzVo2ObPYeElxDCrM7rqzh4+gIAI9qovwskvIQQZvbjr2dRFOgZ5EmQt0ubPY+ElxDCrC71d42MbLtWF0h4CSHMyGhU2GaB/i6Q8BJCmNHhMyUU6qtwd3ZgYGefNn0uCS8hhNlsPVYAwLAIf5wd2zZeJLyEEGZjumSMbNtLRpDwEkKYSXF5NalZRUDbd9aDhJcQwkx+yqhdNbV7Rw9CzLxqakMkvIQQZnGpv2tkGw5MvZyElxCi1YxGhR+OWa6/CyS8hBBmkJZTzLmySjx0jgzu4muR55TwEkK02pajtZeMI3q0/RCJSyS8hBCt9r+j+QCMttAlI0h4CSFaKb+kgkM5JWg0luvvAgkvIUQr/VB3ydgvtAMBnjqLPa+ElxCiVS71d43tablWF0h4CSFaoaLaYNolaEwvCS8hhI3YdaKQi9UGgrxciAr2suhzS3gJIVrsUn/X6J4d0WjaZqONxkh4CSFaRFEU1fq7QMJLCNFCx/JLOV10EZ2jlrgIP4s/v9WE16JFi9BoNDz22GONnrNs2TI0Gk29m4tL2y3wL4Ro3KbDtQNTb+ruj5uzo8Wf3/LP2IC9e/fy3nvvER0d3eS5Xl5eHDt2zPSzpa+zhRC1vv+lNrzGRQWq8vyqt7zKysqYNWsWH3zwAT4+Ta95rdFoCAoKMt0CA9X5wwlhz3KLL5KWU4xGA2N62ml4Pfzww0yePJn4+PhmnV9WVkbnzp0JCwvjlltu4fDhw9c8v7KykpKSkno3IUTrbK5rdQ0I97HoqPrLqRpey5cvJzU1laSkpGadHxkZyUcffcTatWv5/PPPMRqNxMXFcfr06Ubvk5SUhLe3t+kWFhZmrvKFsFub6sLrZpUuGUHF8MrOzmb+/Pl88cUXze50j42NJSEhgf79+zNy5Ei++eYbAgICeO+99xq9T2JiIsXFxaZbdna2uV6CEHappKKaXScKAfX6u0DFDvuUlBQKCgoYMGCA6ZjBYGD79u28/fbbVFZW4uDgcM3HcHJyIiYmhoyMjEbP0el06HTqNGuFaI+2HjtLtUGhW4A7XQM8VKtDtfAaO3YsaWlp9Y7de++99OzZkz/96U9NBhfUhl1aWhqTJk1qqzKFEFf47VvGIFXrUC28PD096dOnT71j7u7u+Pn5mY4nJCQQEhJi6hN78cUXGTp0KBEREVy4cIHFixeTmZnJ3LlzLV6/EPaoqsbI1rpR9WpeMoKVjPNqTFZWFlrtb91yRUVF3H///eTl5eHj48PAgQPZuXMnUVFRKlYphP3YfbKQ0soa/D10xIR1ULUWjaIoiqoVWFhJSQne3t4UFxfj5WXZWfB2Ra8Hj7r+kLIycHdXtx5hFs+uTuOL3VnMHBxG0m1NDypvieZ+RlUf5yWEsA0Go8LGuilBE/oEq1yNhJcQoplSMos4V1aJl4sjsV0tPxH7ShJeQohm2XAoD4D4qECLbW92LepXIISweoqisPFwbXhN6K3uEIlLJLyEEE06eLqYnAsXcXN2YESPALXLASS8hBDNsL7uknF0z464ODU9gNwSJLyEENekKAobDuUCMLGPdVwygoSXEKIJx/JLOVVYjrOj1qI7YjdFwksIcU3r02ovGUd0D8BDZz2TciS8hBDXtN4KLxlBwksIcQ3p+aWk55fh7KAlXuWJ2FeS8BJCNGrdwdpW14ge/ni7OqlcTX0SXkKIBimKwrqDZwCYEt1J5WquJuElhGjQkdxSTpzV4+yoZWwv6/mW8RIJLyFEg75Nq211jY4MwNPFui4ZQcJLCNGA2kvG2v4ua7xkBAkvIUQDDp8pIbOwHBcnLWN6Wt8lI0h4CSEa8N+6jvqxPQNxt6KBqZeT8BJC1KMoCt/WXTJOjlZ/xdTGSHgJIerZn32B00W1y9+MtqK5jFeS8BJC1LN2fw4A43sH4epsHcvfNETCSwhhUmMwmr5lnNbfOr9lvETCSwhh8lPGOQr1Vfi5OzM8wl/tcq5JwksIYfKfA7XfMk6ODsbJwbrjwbqrE0JYzMUqg2mTjVv6h6hcTdMkvIQQAGw+ko++ykCYrysDwjuoXU6TJLyEEACsrbtkvKVfCBqNRuVqmibhJYTgQnkV29ILALjFyr9lvETCSwjBt2m5VBsUooK96B7oqXY5zSLhJYRgVcppAKbH2EarCyS8hLB7J8/pSc26gFYD023gW8ZLJLyEsHPfpNa2ukb0CKCjl4vK1TSfhJcQdsxoVPgmtXYu4+8GhKpczfWR8BLCju06WUjOhYt4ujgyzsq2NmuKhJcQdmxVSm2ra0p0MC5O1ruCREMkvISwU+VVNabdsG3tkhEkvISwWxsO5VFeZeAGPzcGdvZRu5zrJuElhJ1aVfct420DQm1iOtCVJLyEsEPZ58vZkVGIRgO3xtjO2K7LSXgJYYdW7MsGYHiEP2G+bipX0zISXkLYGYNRYUXddKA7BoWpXE3LWU14LVq0CI1Gw2OPPXbN81asWEHPnj1xcXGhb9++fPfdd5YpUIh2YvuvZ8ktrqCDmxM397atsV2Xs4rw2rt3L++99x7R0dHXPG/nzp3MnDmTOXPmsH//fqZPn8706dM5dOiQhSoVwvZ9vbf2knF6/xB0jrY1tutyqodXWVkZs2bN4oMPPsDH59pf17711ltMmDCBhQsX0qtXL1566SUGDBjA22+/baFqhbBthWWVbD6SD8CMG233khGsILwefvhhJk+eTHx8fJPnJicnX3Xe+PHjSU5ObvQ+lZWVlJSU1LsJYa9W78+h2qAQHepNr2AvtctpFUc1n3z58uWkpqayd+/eZp2fl5dHYGD9a/TAwEDy8vIavU9SUhJ//etfW1WnEO2Boigsr7tktPVWF6jY8srOzmb+/Pl88cUXuLi03TIciYmJFBcXm27Z2dlt9lxCWLPUrCIyCspwcdIytZ/tLDrYGNVaXikpKRQUFDBgwADTMYPBwPbt23n77beprKzEwaF+Z2JQUBD5+fn1juXn5xMUFNTo8+h0OnQ6nXmLF8IGfbErC4Cp0Z3wcnFSuZrWU63lNXbsWNLS0jhw4IDpNmjQIGbNmsWBAweuCi6A2NhYtmzZUu/Y999/T2xsrKXKFsImFemrWJdWOwl71tDOKldjHi0Kr08++YRvv/3W9PNTTz1Fhw4diIuLIzMzs1mP4enpSZ8+ferd3N3d8fPzo0+fPgAkJCSQmJhous/8+fPZsGEDS5Ys4ejRo7zwwgvs27ePefPmteRlCGE3VqWepqrGSO9OXvQL9Va7HLNoUXj97W9/w9XVFaj9BnDp0qW8+uqr+Pv78/jjj5utuKysLHJzc00/x8XF8eWXX/L+++/Tr18/Vq5cyZo1a0xhJ4S4mqIofLG79pJx1pDONjkJuyEaRVGU672Tm5sbR48eJTw8nD/96U/k5uby6aefcvjwYUaNGsXZs2fbolazKCkpwdvbm+LiYry8bPurYqum14OHR+2/y8rA3V3deuzYzoxz3Pnhbjx0jux+ZizuOlUHGTSpuZ/RFrW8PDw8KCwsBGDTpk2MGzcOABcXFy5evNiShxRCtJFLra5bY0KsPriuR4teybhx45g7dy4xMTGkp6czadIkAA4fPswNN9xgzvqEEK1QUFrBxsO14yDvHBKucjXm1aKW19KlS4mNjeXs2bOsWrUKPz8/oHb4w8yZM81aoBCi5b7em02NUWFgZx+bH1F/pRa1vDp06NDgfEIZyS6E9ag2GPm8bmzXXUPbV6sLWtjy2rBhAz/99JPp56VLl9K/f3/uvPNOioqKzFacEKLlNh3OJ6+kAn8PZyb1DVa7HLNrUXgtXLjQNME5LS2NJ554gkmTJnHy5EkWLFhg1gKFEC3zyc5TANw5ONyml75pTIsuG0+ePElUVBQAq1atYsqUKfztb38jNTXV1HkvhFDPL2dK2HPqPI5aTbsZUX+lFrW8nJ2dKS8vB2Dz5s3cfPPNAPj6+sqSM0JYgUutrgl9ggj0aruFD9TUopbX8OHDWbBgAcOGDWPPnj189dVXAKSnpxMaanubVwrRnhTpq1hzoHYn7HviblC3mDbUopbX22+/jaOjIytXruSdd94hJKR266T169czYcIEsxYohLg+X+3LprJuHqMtbibbXC1qeYWHh7Nu3bqrjr/xxhutLkgI0XI1BiOfJdcujjA77oZ2M4+xIS2eK2AwGFizZg1HjhwBoHfv3kybNq3BpWyEEJax/lAeORcu4ufuzLR2sODgtbQovDIyMpg0aRI5OTlERkYCtcsth4WF8e2339KtWzezFimEaJqiKHz44wkA7o7tjItT+25ItKjP69FHH6Vbt25kZ2eTmppKamoqWVlZdOnShUcffdTcNQohmmFfZhE/ny7G2VHL3e10eMTlWtTy2rZtG7t27cLX19d0zM/Pj0WLFjFs2DCzFSeEaL5Lra7fDQjBz6P9L33eopaXTqejtLT0quNlZWU4Ozu3uighxPXJLNSz6Zfa/R3mDO+icjWW0aLwmjJlCg888AC7d+9GURQURWHXrl08+OCDTJs2zdw1CiGa8NFPJ1EUGB0ZQERHT7XLsYgWhdc//vEPunXrRmxsLC4uLri4uBAXF0dERARvvvmmmUsUQlzLhfIqvt53GoC5N3VVuRrLafGSOGvXriUjI8M0VKJXr15ERESYtTghRNM+S87kYrWBXsFexHXzU7sci2l2eDW1WsQPP/xg+vfrr7/e8oqEEM12scrAx3XzGB8c2bVdD0q9UrPDa//+/c06z57+eEKo7au9WZzXVxHm68rkdrhm17U0O7wub1kJIdRXbTDywY8nAXhgRDccHVTbQ1oV9vVqhWhH/vvzGXIuXMTfw5n/N9D+VnOR8BLCBhmNCu9uOw7AvcO6tPupQA2R8BLCBm05WkB6fhkeOkfusoOpQA2R8BLCxiiKwtv/+xWAWUPD8XZ1UrkidUh4CWFjtqWf5efTxbg4aZk73H4GpV5JwksIG6IoCm9tqW113TWkMwGe7X8CdmMkvISwITsyCtmfdQGdo5YHRtpvqwskvK6prLKGc2WVapchBHCp1ZUOwJ1Dwuno2T53BWouCa9GrNmfQ2zSFhZvOKZ2KUIAkHyikL2ninB21PLgSFmtWMKrEWG+rpRW1LB6fw4FJRVqlyPsnKIovLW5tq/r9zeGtdu9GK+HhFcjBnb2ZWBnH6oMRtPEVyHUsiOjkN0nz+PsIK2uSyS8ruGBEbUdop/vyqSsskblaoS9UhSFxZtquy/uHBJOpw6uKldkHSS8rmFcr0C6+rtTWlHD8j1Zapcj7NSWIwX8nH0BVycHHh4ta+ZdIuF1DVqthvvrWl8f/XSSaoNR5YqEvTEaFV6ra3XdM+wGux7XdSUJrybcGhOCv4eOM8UVrDt4Ru1yhJ35Ni2Xo3mleOoc+cMI+x7XdSUJrya4ODlwT1ztxNf3tp1AURSVKxL2osZg5I3va8d1zb2pKx3cZGeuy0l4NcNdQzvj5uzA0bxSfjhWoHY5wk6sTDnNiXN6fNycuG/4DWqXY3UkvJqhg5szs4aEA/CPLRnS+hJtrryqhjc217a6Hh4dgaeLfa4ccS0SXs10/4iu6By1HMi+wI6MQrXLEe3cRz+dJL+kklAfV+6Otc/1upqiani98847REdH4+XlhZeXF7Gxsaxfv77R85ctW4ZGo6l3c3GxzEjjjp4uzBxc1/qqW0tJiLZwrqySd7edAGDh+Eh0jva3SmpzqBpeoaGhLFq0iJSUFPbt28eYMWO45ZZbOHz4cKP38fLyIjc313TLzMy0WL1/GNkVZwcte06eZ/cJaX2JtvHPLb9SVllD3xBvpkZ3Urscq6VqeE2dOpVJkybRvXt3evTowSuvvIKHhwe7du1q9D4ajYagoCDTLTAw0GL1Bnu7cvug2o0O/vm/DIs9r7AfJ8/p+WJ37YDoxEk90WplK8HGWE2fl8FgYPny5ej1emJjYxs9r6ysjM6dOxMWFtZkKw2gsrKSkpKSerfWeGhkNxy0Gn7KOEdqVlGrHkuIKy1af4Qao8LoyADiuvmrXY5VUz280tLS8PDwQKfT8eCDD7J69WqioqIaPDcyMpKPPvqItWvX8vnnn2M0GomLi+P06dONPn5SUhLe3t6mW1hYWKvqDfN147aYEADTGBwhzGFnxjk2Hs7HQavh6Ym91C7H6mkUlb/3r6qqIisri+LiYlauXMmHH37Itm3bGg2wy1VXV9OrVy9mzpzJSy+91OA5lZWVVFb+tqBgSUkJYWFhFBcX4+Xl1aKaswrLGbNkKzVGheUPDGVoV78WPU67pteDh0ftv8vKwN1d3XqsXI3ByJR//sTRvFISYjvz4i191C5JNSUlJXh7ezf5GVW95eXs7ExERAQDBw4kKSmJfv368dZbbzXrvk5OTsTExJCR0Xj/k06nM32beenWWuF+bsy4sbYF99rGYzLuS7Ta8r3ZHM0rxdvVicfje6hdjk1QPbyuZDQa67WUrsVgMJCWlkZwcHAbV3W1R8Z0R+eoZV9mEVvTz1r8+UX7UVxezZK6ydePx3fHx12mATWHquGVmJjI9u3bOXXqFGlpaSQmJrJ161ZmzZoFQEJCAomJiabzX3zxRTZt2sSJEydITU3lrrvuIjMzk7lz51q89iBvF+6u2+xzySZpfYmWe2vLrxSVV9O9owez7HQD2ZZwVPPJCwoKSEhIIDc3F29vb6Kjo9m4cSPjxo0DICsrC632t3wtKiri/vvvJy8vDx8fHwYOHMjOnTub1T/WFh4a1Y1/78niUE4JGw7lMbGv5VuAwrYdyyvlk+RTAPxlahRODlZ3MWS1VO+wt7TmdgY215JNx/jn/zLoFuDOxsdG4Cj/8dWSDvsmKYrCjPd2sefUeW6OCuT9hEFql2QVbKbD3tbdP6IrPm5OHD+rZ/nebLXLETbkm9Qc9pw6j6uTA89P6612OTZHwquVvFyceKzu26E3N6dTWlGtckXCFhSXV/O3744AMD++OyGyLv11k/AygzuHhNPF351zZVW8VzehVohrWbzpKIX6Krp39OC+YV3ULscmSXiZgZODlqcn9gTggx9PkFt8UeWKhDU7kH3BNH/xxVv64OwoH8OWkL+amdwcFcjgG3yprDHy2kaZNiQaVlVj5OlVB1GU2v0RYrvJ7IyWkvAyE41GwzOTa+ejfbP/NGmni1WuSFij97Yd52heKb7uzvx5ijpDfNoLCS8z6h/WgVtjQlAU+Mt/DmE02tUoFNGEjIJS01JKz0+NwldG0reKhJeZJU7sibuzA/uzLrAqtfHVLoR9MRoVnl6VRpXByOjIAKb1k0UGW0vCy8w6erkwP747AH/fcJTiizJ0QsAXuzPZl1mEu7MDL9/aF41GFhlsLQmvNnBPXBe6BdQOnXhzs3Te27vMQj1J648C8NSEnjKmy0wkvNqAs6OWF+pGTH+anMnRvNat3ipsl8Go8MTXP1NeZSC2q59pMr9oPQmvNnJT9wAm9gnCYFR4drV03turD388wb7MIjx0jrx6e7SsSW9GEl5t6C9To3B3diAls4gvdltulyNhHY7llbJkU223wZ+n9CLM103litoXCa82FOztylMTakfe/33DMRl5b0eqaows+PoAVQYjY3p25I5Brds7QVxNwquN3TW0MzHhHSirrOH5tdfe6Ui0H69tOsbhMyV0cHNi0W3y7WJbkPBqYw5aDYtui8ZRq2HTL/lsOJSrdkmijW1PP8v722sn6L/6u2g6ellmV3d7I+FlAZFBnjw4shsAf157mCJ9lcoVibZyrqySBV//DMDdQztzc+8glStqvyS8LGTemAgiOnpwtrSS59YeUrsc0QaMRoUnV/zMubJKIgM9eXay7L3YliS8LMTFyYHX7+iHg1bDtwdz+c/PZ9QuSZjZhz+dYOuxs+gctfxjZgwuTg5ql9SuSXhZUHRoB+aNjgDgz2sOkV9SoXJFwlx2nyjk7xtqty/7y9QoIoM8Va6o/ZPwsrB5YyLoG+JN8cVq/rTqoGyZ1g4UlFQw79/7MRgVbo0J4c7B4WqXZBckvCzMyUHL63f0w9lRy9ZjZ/k0WQav2rIag5F5/97P2dJKegR68MqtfWRYhIVIeKmge6AniXXLRr/y7REOn5GFC23V3zccZc/J83joHHnnroG4Oau6FapdkfBSyT1xNxDfK5Aqg5FHvtyPvrJG7ZLEdVqVcpoPfjwJwKu3R9MtwEPliuyLhJdKNBoNi2+PJtjbhRPn9PxZhk/YlNSsIhK/SQNg3ugIJslu6RYn4aUiH3dn3vp9DFpN7QakK/bJprW2ILf4In/4LIUqg5FxUYEsGNdD7ZLskoSXygZ38eXxuk1rn1tzSDbusHLlVTU88GkKZ0trB6K+MaO/LHOjEgkvK/Dw6AjG9uxIZY2RBz9PobCsUu2SRANqDEYe/fd+0nKK8XFz4sPZg/DQSQe9WiS8rIBWq+H1Gf3p4u9OzoWLzPtyPzUGo9plicsoisIL/z3M5iMFODtq+SBhkKzPpTIJLyvh7erE+3cPxN3ZgeQThfztu6NqlyQu8+62E3y+KwuNBt6a0Z9BN/iqXZLdk/CyIt0DPVlyRz8APtpxUlZftRLfpJ7m7xtq/2fy58lRTJRvFq2ChJeVmdAn2NSB/5e1h9l6rEDliuzbhkN5LFx5EIA5w7tw3/AuKlckLpHwskKPjo3gdwNCMRgVHv4ilV/OyO5DatiefpZH6+Ys3j4wlGcnyRI31kTCywppNBqSbutLbFc/9FUG7lu2lzMXZP17S9p76jwPfLaPKoORSX2DWHRbXxkSYWUkvKyUs6OWd+8eSERHD/JKKrjrw92ckyEUFrHn5Hnu+WgPFdVGRkUG8OaMGBwd5KNibeQdsWLerk58et9gQjq4cuKcnoR/7aH4YrXaZbVryccLmf3RHvRVBoZF+PHOrIE4O8rHxBrJu2LlOnVw5fO5Q/D30PFLbgn3LdtLeZVM4m4LOzLOce+yPVysNnBTd3/+NftGXJ1lNVRrJeFlA7r4u/PZnMF4uTiSklnEfcv2yioUZvb9L/nct2wvFdVGRkcG8EHCIFnG2cpJeNmIXsFefHLfYDx0juw6cZ6Ej/ZQUiGXkOawfE8Wf/hsH5U1RuJ7BfLu3QMluGyAhJcNiQn34fO5Q0wtsLs/3E1xuQRYSymKwj+3/MrT36RhVOCOQaG8e9cAdI4SXLZAwsvG9A/rwJf3D8XHzYmfTxfz+w92yUYeLVBVY+SZ1Wks+T4dgIdHd+Pvv4uWbxVtiKrv1DvvvEN0dDReXl54eXkRGxvL+vXrr3mfFStW0LNnT1xcXOjbty/fffedhaq1Hn1CvFn+QCz+HjqO5JZw2//tJD2/VO2ybEaRvoq7/7Wbf+/JRqOB56dGsXB8T1l73saoGl6hoaEsWrSIlJQU9u3bx5gxY7jllls4fPhwg+fv3LmTmTNnMmfOHPbv38/06dOZPn06hw7Z3yqkkUGefPNQHF3rVqL43Ts7ST5eqHZZVi+joJTp/7eD3XXrzv9r9iDuHSZTfmyRRrGyvbd8fX1ZvHgxc+bMuep3M2bMQK/Xs27dOtOxoUOH0r9/f959990GH6+yspLKyt8Gd5aUlBAWFkZxcTFeXl7mfwEWVqSvYu6n+0jJLMLZQcvL0/twx41hapcFej141K3pXlYG7u7q1gP85+czJK46iL7KQJivK/+afSM9AmV/RWtTUlKCt7d3k59Rq7nANxgMLF++HL1eT2xsbIPnJCcnEx8fX+/Y+PHjSU5ObvRxk5KS8Pb2Nt3Cwqzgg21GPu7OfDF3CJP6BlFlMPLUqoM8uzqNqhpZD+ySyhoDz61J49F/70dfZWBoV1/WPjxcgsvGqR5eaWlpeHh4oNPpePDBB1m9ejVRUVENnpuXl0dgYGC9Y4GBgeTl5TX6+ImJiRQXF5tu2dntb514FycH3p45gCfG9UCjgS92Z/H795PJK5aO/ONny7j9nWQ+35UFwCNjIvh8zhB83Z1Vrky0lupr2EZGRnLgwAGKi4tZuXIls2fPZtu2bY0G2PXS6XTodDqzPJY102o1PDK2O31CvJm/fD+pWReY+NZ2Fv0umvG9g9Quz+KMRoVPkk+xaP1RKmuM+Lg58caM/oyK7Kh2acJMVG95OTs7ExERwcCBA0lKSqJfv3689dZbDZ4bFBREfn5+vWP5+fkEBdnfh7Mxo3t25D/zhtO7kxdF5dX84bMUnl510K5G5GefL+euf+3mr//9hcoaIzd19+e7+TdJcLUzqofXlYxGY70O9svFxsayZcuWese+//77RvvI7NUN/u6s/uMw/jCyKxoNLN+bzaR//MiOjHNql9amqmqMvLP1OOPe2MbO44W4Ojnw0vQ+fHrfYIK9XdUuT5iZqpeNiYmJTJw4kfDwcEpLS/nyyy/ZunUrGzduBCAhIYGQkBCSkpIAmD9/PiNHjmTJkiVMnjyZ5cuXs2/fPt5//301X4ZVcnbUkjixFyN7BPDE1z+TWVjOrA93c1tMCM9O7oWfR/u6lE4+Xshf1h7i14IyAIZ08WXR76Lp4q/+t5yibagaXgUFBSQkJJCbm4u3tzfR0dFs3LiRcePGAZCVlYVW+1vjMC4uji+//JLnnnuOZ555hu7du7NmzRr69Omj1kuwenHd/Nn4+AiWbDzGp7sy+WZ/Dv87VsD8sd2ZNaSzzS/3kp5fyt/XH2XL0drlsv3cnXl2ci9ujQmRQaftnNWN82przR1D0h7tr9ui/mhe7Wj8cF83Fo6PZEp0sPk/6G08ziuzUM///XCcFSnZGBVw0Gq4c3A4T94cibebk1mfS1hWcz+jEl52psZg5Kt92bzx/a+mlVl7BXvx4MiuTO4bbL65fW0UXkdyS3hn63HWHTyDse6/3Il9glg4PpKuAR5meQ6hLgmvRth7eF2ir6zhwx9P8v724+irDACE+boyZ1gXbo0JbX3rxYzhVVVj5Ptf8vlyTyY7Mn6bAjUqMoBHx3ZnQLhP62oVVkXCqxESXvVdKK/i0+RMlu08xXl9FQA6Ry0T+wRxx41hDO3i17KNJ1oZXoqikJZTzLcHc1mVeppzZbW1aTQwqW8wD43sRp8Q7+uvS1g9Ca9GSHg17GKVgZUp2Xy+K4tjl61Q4e+hY1xUR26OCiK2m1/zF+lrQXhVVBvYd6qI7b+eZf2hXLLP/7ZjUoCnjhmDwphxYxhhvm7X9dqEbZHwaoSE17UpisLPp4v5am826w6eobTit8GtTg4aokM7MOgGHwaG+9Aj0JMwXzccGmqZNRFeiqKQff4ih88Uc+hMMamZF0jJKqo3J9PVyYExvToyNboTY3t1xEnW2rILEl6NkPBqvqoaI7tPFvL9L/l8/0s+uQ3MlXR21NLV350gbxf8PXQEeOrw0DniUnmROROjAVj6nwMUOzhTWlHNmQsV5Fy4SE7RRS5WG656vCAvF+Ii/IjvFcjoyI6yAYYdkvBqhIRXyyiKQtb5cvaeKmLvyfOk5RRz/GwZlY2sXuFaVcGRN24HoNfjK7no7HLVOc4OWiKDPOndyYs+Id7EdvOjq7+7jM+yc839jKo+MVvYBo1GQ2c/dzr7uXP7wFAADEaF00XlnDirp6C0gnNlVZwtreRilQFNud503/83KARnLy88XBwJ9nYhpIMboT6udOrgavODZIV6pOUl2oYVLkYobIPNLUYohBDXQ8JLCGGTJLyEEDZJwksIYZMkvIQQNknCSwhhkyS8hBA2ye4GqV4a1lZSUqJyJe2c/rdBqpSUgOHqqUBCNOTSZ7OpIah2F16lpbUrJrS3zWetWqdOalcgbFBpaSne3o0ve2R3I+yNRiNnzpzB09PTLufQlZSUEBYWRnZ2tswwsAH2+H4pikJpaSmdOnWqt4fFleyu5aXVagkNDVW7DNV5eXnZzYehPbC39+taLa5LpMNeCGGTJLyEEDZJwsvO6HQ6nn/+eXS69rXpbHsl71fj7K7DXgjRPkjLSwhhkyS8hBA2ScJLCGGTJLyEEDZJwksIYZMkvEQ9b7zxBr179yYqKopHH320ycmxwrJuvfVWfHx8uP322+sdX7duHZGRkXTv3p0PP/xQpeosS4ZKCJOzZ88ydOhQDh8+jJOTEyNGjOC1114jNjZW7dJEna1bt1JaWsonn3zCypUrAaipqSEqKooffvgBb29vBg4cyM6dO/Hz81O52rYlLS9RT01NDRUVFVRXV1NdXU3Hjh3VLklcZtSoUXh6etY7tmfPHnr37k1ISAgeHh5MnDiRTZs2qVSh5Uh4tSPbt29n6tSpdOrUCY1Gw5o1a646Z+nSpdxwww24uLgwZMgQ9uzZY/pdQEAATz75JOHh4XTq1In4+Hi6detmwVfQvrX2/WnMmTNnCAkJMf0cEhJCTk6OOUu3ShJe7Yher6dfv34sXbq0wd9/9dVXLFiwgOeff57U1FT69evH+PHjKSgoAKCoqIh169Zx6tQpcnJy2LlzJ9u3b7fkS2jXWvv+iPokvNqRiRMn8vLLL3Prrbc2+PvXX3+d+++/n3vvvZeoqCjeffdd3Nzc+OijjwDYvHkzERER+Pr64urqyuTJk9m1a5clX0K71tr3pzGdOnWq19LKycmhkx0sACnhZSeqqqpISUkhPj7edEyr1RIfH09ycjJQu7rszp07qaiowGAwsHXrViIjI9Uq2a405/1pzODBgzl06BA5OTmUlZWxfv16xo8f39Ylq87uFiO0V+fOncNgMBAYGFjveGBgIEePHgVg6NChTJo0iZiYGLRaLWPHjmXatGlqlGt3mvP+AMTHx/Pzzz+j1+sJDQ1lxYoVxMbGsmTJEkaPHo3RaOSpp55q9980goSXuMIrr7zCK6+8onYZohGbN29u8Pi0adPs7n80ctloJ/z9/XFwcCA/P7/e8fz8fIKCglSqSlwi78/1k/CyE87OzgwcOJAtW7aYjhmNRrZs2SKDUK2AvD/XTy4b25GysjIyMjJMP588eZIDBw7g6+tLeHg4CxYsYPbs2QwaNIjBgwfz5ptvotfruffee1Ws2n7I+2Nmimg3fvjhBwW46jZ79mzTOf/85z+V8PBwxdnZWRk8eLCya9cu9Qq2M/L+mJfMbRRC2CTp8xJC2CQJLyGETZLwEkLYJAkvIYRNkvASQtgkCS8hhE2S8BJC2CQJLyGETZLwEkLYJAkvYdVGjRrFY4891uzzly1bRocOHVr8fI2tLS+sj0zMFuIyubm5+Pj4qF2GaAYJLyEuI2tn2Q65bBTXbdSoUTzyyCM89thj+Pj4EBgYyAcffGBavsXT05OIiAjWr19f737btm1j8ODB6HQ6goODefrpp6mpqTH9Xq/Xk5CQgIeHB8HBwSxZsuSq566srOTJJ58kJCQEd3d3hgwZwtatW5tde1VVFfPmzSM4OBgXFxc6d+5MUlKS6feXXza+8MILaDSaq27Lli0DatfbSkpKokuXLri6utKvXz/TRrDCAtRe1kLYnpEjRyqenp7KSy+9pKSnpysvvfSS4uDgoEycOFF5//33lfT0dOWhhx5S/Pz8FL1eryiKopw+fVpxc3NT/vjHPypHjhxRVq9erfj7+yvPP/+86XEfeughJTw8XNm8ebNy8OBBZcqUKYqnp6cyf/580zlz585V4uLilO3btysZGRnK4sWLFZ1Op6SnpyuKoigff/yx4u3t3WjtixcvVsLCwpTt27crp06dUn788Uflyy+/NP0eUFavXq0oiqKUlpYqubm5pttrr72muLm5KWlpaYqiKMrLL7+s9OzZU9mwYYNy/Phx5eOPP1Z0Op2ydetW8/yhxTVJeInrNnLkSGX48OGmn2tqahR3d3fl7rvvNh3Lzc1VACU5OVlRFEV55plnlMjISMVoNJrOWbp0qeLh4aEYDAaltLRUcXZ2Vr7++mvT7wsLCxVXV1dTeGVmZioODg5KTk5OvXrGjh2rJCYmKorSdHg98sgjypgxY+rVcbnLw+tyycnJiouLi/LVV18piqIoFRUVipubm7Jz5856582ZM0eZOXNmo88vzEf6vESLREdHm/7t4OCAn58fffv2NR27tAvOpQ1Tjxw5QmxsLBqNxnTOsGHDKCsr4/Tp0xQVFVFVVcWQIUNMv/f19a239VpaWhoGg4EePXrUq6WysrLZu+Xcc889jBs3jsjISCZMmMCUKVO4+eabr3mfrKwspk+fzpNPPskdd9wBQEZGBuXl5YwbN67euVVVVcTExDSrFtE6El6iRZycnOr9rNFo6h27FFJGo9Fsz1lWVoaDgwMpKSk4ODjU+52Hh0ezHmPAgAGcPHmS9evXs3nzZu644w7i4+Mb7avS6/VMmzaN2NhYXnzxxXq1AHz77beEhITUu49Op7uelyVaSMJLWESvXr1YtWoViqKYgm3Hjh14enoSGhqKr68vTk5O7N69m/DwcACKiopIT09n5MiRAMTExGAwGCgoKOCmm25qcS1eXl7MmDGDGTNmcPvttzNhwgTOnz+Pr69vvfMUReGuu+7CaDTy2Wef1Ws1RkVFodPpyMrKMtUnLEvCS1jEH//4R958800eeeQR5s2bx7Fjx3j++edZsGABWq0WDw8P5syZw8KFC/Hz86Njx448++yzaLW/fSHeo0cPZs2aRUJCAkuWLCEmJoazZ8+yZcsWoqOjmTx5cpN1vP766wQHB5s21l2xYgVBQUENDmx94YUX2Lx5M5s2baKsrMzU2vL29sbT05Mnn3ySxx9/HKPRyPDhwykuLmbHjh14eXkxe/Zss/3tRMMkvIRFhISE8N1337Fw4UL69euHr68vc+bM4bnnnjOds3jxYsrKypg6dSqenp488cQTFBcX13ucjz/+mJdffpknnniCnJwc/P39GTp0KFOmTGlWHZ6enrz66qv8+uuvODg4cOONN/Ldd9/VC8lLtm3bRllZGXFxcVfVcM899/DSSy8REBBAUlISJ06coEOHDgwYMIBnnnmmBX8hcb1kAw4hhE2SQapCCJsk4SWEsEkSXkIImyThJYSwSRJeQgibJOElhLBJEl5CCJsk4SWEsEkSXkIImyThJYSwSRJeQgib9P8BG5GIwiV70WYAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEqCAYAAABEE9ZrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuRElEQVR4nO3dd3xUVf7/8dekTdpMGpCEFEACCQETQhRIQEAFqVJ2XV0WBV3Erx1FccV1v7qiht8itoW14E/UXVlAFNgfRdpSFEJvoQiElkIKhJDJJKTN3N8fQ0aiBEKYzJ3yeT4e9/FI7tyZ+SRh3px77rnnaBRFURBCCCfjoXYBQgjRHBJeQginJOElhHBKEl5CCKck4SWEcEoSXkIIpyThJYRwSl5qF2BvZrOZs2fPotPp0Gg0apcjhPgFRVEoLy+nbdu2eHg03r5yu/A6e/YsMTExapchhLiO3NxcoqOjG33c7cJLp9MBll+MXq9XuRoXVlEBbdtavj57FgIC1K1HOA2DwUBMTIz1s9oYtwuv+lNFvV4v4dWSPD1//lqvl/ASN+x63TrSYS+EcEoSXkIIpyThJYRwShJeQginJOElhHBKEl5CCKck4SWEaHGf/3iKZfvyMVTV2uw1JbyEEC2q1mRm1pqjTF6wj5ySSpu9roSXEKJF7cu9SEWNidAAHxIjbTcwXMJLCNGifjh+HoD0jmF4eNhuMgQJLyFEi/rx+DkA7ujUyqavK+ElhGgxhqpa9ueVAdC3U2ubvraElxCixWSeKMFkVrilVQBRwX42fW0JLyFEi/nxcn9XXxufMoKElxCiBf2YbQmvPnESXkIIJ5FXWsmp8xV4emhI6xhm89eX8BJCtIgtl1tdydFB6H29bf76El5CiBbxg7W/y7ZXGetJeAkhbM5sVth6ogSw/fiuehJeQgibO1xg4EJFDYFaL7rHBLfIe0h4CSFsbtMxy6j63reE4u3ZMjEj4SWEsLn68Oof36bF3kPCSwhhU+VVtew5UwpA/xbqrAcJLyGEjW09UULd5VuCYsP8W+x9JLyEEDZVf8rYr3PLtbpAwksIYUOKorDpaH1/l4SXEMJJnDhXQf7FS/h4edC7g+1vCbqShJcQwmbqTxl7dQjFz8ezRd9LwksIYTPWIRIt3N8FEl5CCBupqjWx/aTllqABLdzfBRJeQggb2XayhOo6M1HBfnRsHdji7yfhJYSwiSuHSGg0tlslqDESXkIIm9jwUzFgn/4ucKDwmjFjBhqNhueee67RY7744gs0Gk2DzdfX135FCiGu6uQ5I6dLKvH21LTIfPVX42WXd7mOnTt38sknn5CUlHTdY/V6PUePHrV+b4/mqRDi2v57udXV+5YwArX2iRXVW15Go5Fx48Yxd+5cQkJCrnu8RqMhIiLCuoWHh9uhSiHEtdSH150tOIvEL6keXk899RTDhw9n4MCBTTreaDTSrl07YmJiGDVqFIcOHbrm8dXV1RgMhgabEMJ2yqtq2XHqAgB3JbhJeC1YsIA9e/aQkZHRpOPj4+P5/PPPWbZsGf/6178wm82kp6eTl5fX6HMyMjIICgqybjExMbYqXwiBZW3G+lkk2rcKsNv7qhZeubm5TJ48ma+//rrJne5paWmMHz+e7t27079/f7777jtat27NJ5980uhzpk2bRllZmXXLzc211Y8ghODnU0Z7trpAxQ773bt3U1xcTI8ePaz7TCYTmzdvZvbs2VRXV+Ppee17o7y9vUlJSSE7O7vRY7RaLVqt1mZ1CyF+ZjYrbDjqZuF19913k5WV1WDfI488QkJCAn/605+uG1xgCbusrCyGDRvWUmUKIa4hK7+M80bLQhu3tQ+163urFl46nY5u3bo12BcQEEBYWJh1//jx44mKirL2ib3xxhv07t2buLg4Ll68yMyZMzlz5gyPPvqo3esXQvx8ynhHp1b4eNm3F8ohxnk1JicnBw+Pn38hpaWlTJo0icLCQkJCQkhNTWXr1q0kJiaqWKUQ7kutU0YAjaIoit3fVUUGg4GgoCDKysrQ6/Vql+O6Kiog8PLNuUYjBNjvKpSwj2JDFT3fXo9GAzteGUhrnW36lpv6GVV9nJcQwjmtO2JpdXWPCbZZcN0ICS8hRLOsPVwIwMAu6tzlIuElhLhhFdV1bDlhmXjwnkQJLyGEk/jh+Dlq6sy0C/Mnrk3LTzx4NRJeQogbtuZwEQCDuoSrNrOLhJcQ4obUmczWiQcHqnTKCBJeQogbtPtMKaWVtQT7e3Nbu+tPY9VSJLyEEDdk7eVTxrvi2+DlqV6ESHgJIZpMURTWHrnc36XiKSNIeAkhbkB2sZEzJZX4eHpwh50W2miMhJcQosnqrzKmdbTfXPWNkfASQjTZ9wcto+qHdItQuRIJLyFEE+WVVpKVX4aHRv3+LpDwEkI00epDllPG29uH0ipQ/dmJJbyEEE3y/cECwDFOGUHCSwjRBMXlVew6UwrA4K4SXkIIJ7H2cBGKAskxwbQN9lO7HEDCSwjRBNarjA7S6gIJLyHEdZRV1pJ5ee6uwV3Vv8pYT8JLCHFN644UUWdWiA/XcUtrdebuuhoJLyHENa26fMo42EGuMtaT8BJCNKq8qpbNx84BMFTCSwjhLNYdKaLGZKZj6wASInRql9OAhJcQolErDlgGpg5PaqvadM+NkfASQlxV2aVaNh87D8CIpEiVq/k1CS8hxFWtO2w5ZezUJpDO4Y51yggSXkKIRqzIqj9ldLxWF0h4CSGuoqyylh+OW64yDr9VwksI4STWHC6k1mQZmNrJAU8ZQcJLCHEV9aeMwxy01QUSXkKIX7hYWcOPxy1XGYcnOdbA1CtJeAkhGliZVUidWSEhQkdcG8c8ZQQJLyHELyzblw/A6JQolSu5NgkvIYTV2YuX2HH6AgD3JrdVuZprk/ASQlgtP3AWRYGe7UOJcpAZUxsj4SWEsPrP/rMAjOzu2K0ukPASQlyWXWzkYL4BLw+NQw+RqCfhJYQAfm519evcmtAAH5WruT4JLyEEiqLwn8tXGUc5wSkjSHgJIYADeWWcLqnEz9uTgV0cZ5GNa5HwEkKwZK+l1TUoMZwArZfK1TSNhJcQbq7WZLb2d43p4dgDU68k4SWEm9t49BwXKmpordNyR1wrtctpMgkvIdzcd3vyABjdvS1ens4TCQ5T6YwZM9BoNDz33HPXPO6bb74hISEBX19fbr31VlauXGmfAoVwQRcra1h/pBiA3/SIVrmaG+MQ4bVz504++eQTkpKSrnnc1q1bGTt2LBMnTmTv3r2MHj2a0aNHc/DgQTtVKoRrWX6ggBqTmS6RerpE6tUu54aoHl5Go5Fx48Yxd+5cQkJCrnnsBx98wJAhQ5g6dSpdunRh+vTp9OjRg9mzZzf6nOrqagwGQ4NNCGFRf8r4WyfqqK+neng99dRTDB8+nIEDB1732MzMzF8dN3jwYDIzMxt9TkZGBkFBQdYtJibmpmsWwhWcPGdkT85FPD00TnEv4y+pGl4LFixgz549ZGRkNOn4wsJCwsMbDqALDw+nsLCw0edMmzaNsrIy65abm3tTNQvhKurHdvXr1Io2Ol+Vq7lxqo1Gy83NZfLkyaxduxZf35b7xWm1WrRabYu9vhDOyGRW+G6PJbzGOFlHfT3Vwmv37t0UFxfTo0cP6z6TycTmzZuZPXs21dXVeHp6NnhOREQERUVFDfYVFRUREeG482wL4Yi2ZJ8n/+Il9L5e3JPoHLcD/ZJqp4133303WVlZ7Nu3z7rddtttjBs3jn379v0quADS0tJYv359g31r164lLS3NXmUL4RIW7bJ0n4xOicLX+9efNWegWstLp9PRrVu3BvsCAgIICwuz7h8/fjxRUVHWPrHJkyfTv39/Zs2axfDhw1mwYAG7du3i008/tXv9Qjir0ooa1hyynMHcf5vzXsBS/WrjteTk5FBQUGD9Pj09nfnz5/Ppp5+SnJzM4sWLWbp06a9CUAjRuGX78qkxmUmM1NMtKkjtcppNoyiKonYR9mQwGAgKCqKsrAy93rkG5TmVigoIDLR8bTRCQIC69QjAMm/XsA9/5EiBgb+O7MqE9PZql/QrTf2MOnTLSwhhW4fOGjhSYMDHy8NpJh1sjISXEG5k4U5LR/3grhEE+zv+VM/XIuElhJuoqjVZF5R9wIk76utJeAnhJlYcKMBQVUdUsB/pHcPULuemSXgJ4Sbm78gB4A+9YvHw0Khczc1rVnh9+eWXrFixwvr9Sy+9RHBwMOnp6Zw5c8ZmxQkhbOOnQgO7z5Ti5aHhd7c55+1Av9Ss8Hr77bfx87MsBZ6ZmcmcOXP429/+RqtWrXj++edtWqAQ4ubN325pdQ1KDHfKm7Cvplkj7HNzc4mLiwNg6dKl/Pa3v+Wxxx6jT58+DBgwwJb1CSFuUmVNHUsu34Q9rlc7lauxnWa1vAIDAykpKQFgzZo1DBo0CABfX18uXbpku+qEEDft/+0/S3l1He3C/F2io75es1pegwYN4tFHHyUlJYVjx44xbNgwAA4dOkT79u1tWZ8Q4iZ9ffmU8Q89XaOjvl6zWl5z5swhLS2Nc+fO8e233xIWZknz3bt3M3bsWJsWKIRovqy8Mg7kleHj6cF9qa7RUV+vWS2v4ODgq84b/9e//vWmCxJC2M6/tlmu/g/uFkFYoGtNytmsltf333/Pjz/+aP1+zpw5dO/enT/84Q+UlpbarDghRPNdrKxh6eUR9ePTXKejvl6zwmvq1KnWVXiysrJ44YUXGDZsGKdOnWLKlCk2LVAI0TyLduVSXWeZ+ua2dtdemcsZNeu08dSpUyQmJgLw7bffMmLECN5++2327Nlj7bwXQqjHZFb4KtNyyjghvR0ajet01NdrVsvLx8eHyspKANatW8c999wDQGhoqKyLKIQD2PBTMXmllwjy82ZksvOtydgUzWp59e3blylTptCnTx927NjBwoULATh27BjR0a51RUMIZ/Rl5mkAfn97DH4+zjlH/fU0q+U1e/ZsvLy8WLx4MR999BFRUZZkX7VqFUOGDLFpgUKIG5NdbOSH4+fRaODB3q7XUV+vWS2v2NhYli9f/qv977333k0XJIS4OfXDI+5OaENMqL/K1bScZq8eZDKZWLp0KUeOHAGga9eujBw58qpLlgkh7MNQVcs3l5c1G5/WXt1iWlizwis7O5thw4aRn59PfHw8ABkZGcTExLBixQo6duxo0yKFEE2zcEcuFTUmOrUJ5I5OrdQup0U1q8/r2WefpWPHjuTm5rJnzx727NlDTk4OHTp04Nlnn7V1jUKIJqgzmZm35RQAj97RwSWHR1ypWS2vTZs2sW3bNkJDQ637wsLCmDFjBn369LFZcUKIplt1sJCzZVWEBfgwqrtrDo+4UrNaXlqtlvLy8l/tNxqN+Pg494okQjgjRVH47EdLq+vB3u3w9Xb9vudmhdeIESN47LHH2L59O4qioCgK27Zt4/HHH2fkyJG2rlEIcR17ckrZn3sRHy8Plx4ecaVmhdeHH35Ix44dSUtLw9fXF19fX9LT04mLi+P999+3cYlCiOv57AdLq2tM9yha61xr9ojGNHtKnGXLlpGdnW0dKtGlSxfr1NBCCPvJKalk9aFCACbe0UHlauynyeF1vdkiNmzYYP363XffbX5FQogbMveHk5gV6Ne5NZ3DdWqXYzdNDq+9e/c26ThXvzwrhCM5b6xm0eVBqU/0d6/xlU0OrytbVkIIx/DFltNU15npHhNM71tCr/8EFyIrZgvhpIzVdXx1efaIx/t3dLuzHgkvIZzUv7fnYKiq45bWAdyTGK52OXYn4dWI7GIj0747YP2fTQhHUl1n4rMfTwLweL+OLrWkWVNJeDVi1+kL/HtHLp9uPonJrKhdjhANLN2bT5Ghmgi9L6NS2qpdjiokvBoxOiWKEH9v8kovsfZwkdrlCGFVZzLzj40nAJjYtwNaL9e/FehqJLwa4evtydiesQDWO/WFcAT/2X+WMyWVhAb4MK53rNrlqEbC6xoeSmuHp4eG7acucOhsmdrlCIHJrDD7v9mAZdobf59mzyfq9CS8riEyyI+h3SIAmLfltLrFCAEsP3CWk+crCPb3dvmZUq9Hwus6HuljuVfsP/vOct5YrXI1wp2Zr2h1TezTgUCt+7a6QMLrunrEBpMcHUSNycz87TlqlyPc2PeHCjlebETn68WEPu3VLkd1El7XodFo+GNfS+vrn9vOUFNnVrki4Y7MZoUP1x8HLGcDel9vlStSn4RXEwztFkm4Xsu58mqW7ctXuxzhhlZkFfBTYTk6rRd/lFYXIOHVJD5eHvzxct/Xp5tPYpZBq8KO6kxm3lt3DIBH77iFYH+Zah1UDq+PPvqIpKQk9Ho9er2etLQ0Vq1a1ejxX3zxBRqNpsHm6+trl1rH9opFp/XieLGRDUeL7fKeQgAs3XeWk+csVxj/2Le92uU4DFXDKzo6mhkzZrB792527drFXXfdxahRozh06FCjz9Hr9RQUFFi3M2fO2KVWva83f7g8IPCTTSft8p5C1NSZ+WC9pdX1eP+O6KSvy0rV8Lr33nsZNmwYnTp1onPnzrz11lsEBgaybdu2Rp+j0WiIiIiwbuHh9rub/o99OuDtqWHH6QvsPlNqt/cV7mvRrlxyL1yiVaCWCW4+ruuXHKbPy2QysWDBAioqKkhLS2v0OKPRSLt27YiJibluKw2guroag8HQYGuucL0vY1Is6+F9sulEs19HiKaoqjVZx3U9fWdH/Hzc8x7GxqgeXllZWQQGBqLVann88cdZsmQJiYmJVz02Pj6ezz//nGXLlvGvf/0Ls9lMeno6eXl5jb5+RkYGQUFB1i0mJuam6n2s3y0ArD1SRHax8aZeS4hr+SrzNIWGKtoG+TK2l/vew9gYjaIoql46q6mpIScnh7KyMhYvXsxnn33Gpk2bGg2wK9XW1tKlSxfGjh3L9OnTr3pMdXU11dU/j4w3GAzExMRQVlaGXq9vVs2TvtrF2sNF3JcazTu/S27Wa7i8igoIDLR8bTRCQIC69TiZi5U19PvbBgxVdcy8L4nf3XZz/+k6E4PBQFBQ0HU/o6q3vHx8fIiLiyM1NZWMjAySk5P54IMPmvRcb29vUlJSyM7ObvQYrVZrvZpZv92sp+60LPG2ZG8+OSWVN/16QvzSPzaewFBVR0KEjt/0iFa7HIekenj9ktlsbtBSuhaTyURWVhaRkZEtXFVD3WOC6de5NSazwj82Nh6cQjRHXmklX1yeCOBPQxPwdMNZUptC1fCaNm0amzdv5vTp02RlZTFt2jQ2btzIuHHjABg/fjzTpk2zHv/GG2+wZs0aTp48yZ49e3jwwQc5c+YMjz76qN1rn3y3pfW1eHceeaXS+hK28+6aY9SYzKR3DGNA59Zql+OwVL0tvbi4mPHjx1NQUEBQUBBJSUmsXr2aQYMGAZCTk4OHx8/5WlpayqRJkygsLCQkJITU1FS2bt3apP4xW0ttF0qfuDC2ZJfw0cYTvDXmVrvXIFzPobNlLLl8C9q0oV3cbkWgG6F6h729NbUzsCm2nyzhgU+34e2pYdPUO2kb7GejKl2AdNjfMEVRGPfZdraeKGFkcls+HJuidkmqcJoOe2fW65YwenUIpdak8LGM+xI3ac3hIraeKMHHy4Opg+PVLsfhSXjdpMl3dwJgwY5c6fsSzVZdZ+LtlUcAmHRHB2JC/VWuyPFJeN2ktI5hpN0SRo3JbJ1vSYgbNW/Lac6UVNJGp+XJAXFql+MUJLxukkajYeoQSxN/8e48TpyTUffixpwrr7beBvSnIQkEuPn0zk0l4WUDPWJDGNglHLMC7649pnY5wsm8s/ooxuo6kmOCrffOiuuT8LKRFwd3RqOBFQcKOJgvy6SJptmbU8qi3bkA/O+IRDxkQGqTSXjZSEKEnlHJlmXX31lzVOVqhDMwmRX+suwgigK/7RFNarsQtUtyKhJeNvT8oM54eWjYePQcmSdK1C5HOLivt5/hYL4Bva8X04YlqF2O05HwsqF2YQH84fLUJW+tPCxz3YtGnSuvZuZqSwt96pAEWgVqVa7I+Uh42djkuzuh03pxMN/Akr2y0pC4uoyVRyivqiMpOog/9JS5uppDwsvGwgK1PH2XZZzOzNVHqaypU7ki4WgyT5Tw3d58NBqYPqqbzBrRTBJeLWBCenuiQ/woNFQxd/MptcsRDqSq1sS07w4AMK5XLMkxweoW5MQkvFqAr7cnLw+1dMB+vOkERYYqlSsSjuL9dcc5XVJJhN6Xl4ZIJ/3NkPBqIcNvjaRHbDCXak38n1U/qV2OcAAH88uY+4Nl2bzpo7uhl2XMboqEVwvRaDT8771d0Wjgu7357Dx9Qe2ShIrqTGZe/u4AJrPC8KRIBiXab8k+VyXh1YK6xwTz+9stCyf8ZelB6kxmlSsSapn7wykO5hsI8vPm9Xu7ql2OS5DwamFTBycQ7O/NT4Xl/HObfVb3Fo7lWFE5712+5/XV4V1orZMxXbYg4dXCQgN8rBPLvbvmGMXl0nnvTmpNZqYs2keNycxdCW24L1VWArIVCS87+P3tsSRFB1FeXceMldJ5705m/zebg/kGgv29mfGbW2VOehuS8LIDTw8Nb4zqZu2833zsnNolCTs4kHeR2Rss83RNH9WNNnpflStyLRJedtI9JpgJae0BeGVJFhXVMvLelV2qMTFl0X5MZoURSZHce3nGEWE7El52NHVwPFHBfuSVXmLWGpm00JW9ueIw2cVGWuu0TB/VTe1yXJKElx0FaL14a4zlH/K8rafYm1OqckWiJXx/sJCvt+eg0cB793cnJMBH7ZJckoSXnQ2Ib8NvUqJQFHj52yxq6mTslys5e/ESf/rWcu/iY/1uoW+nVipX5LokvFTwlxGJhAX4cLSonPfXyemjqzCZFZ5buI+yS7UkRwfxwiBZe7ElSXipICTAx3r6+PGmE+ySW4dcwgfrjrHj1AUCfDz5cGwKPl7y8WpJ8ttVyZBukfy2RzRmBaYs2o9Rrj46tQ0/FfPh5eXL3hpzK+3CAlSuyPVJeKnotZGJRAX7kXOhkjeXH1a7HNFMuRcqeW7hPgAe6t2O0bJ8mV1IeKlI7+vNrPuT0Whgwc5c1hwqVLskcYOqak08+fUeSz9XTDCvjuiidkluQ8JLZb1vCWPSHbcA8NK3B8i/eEnlikRTKYrC6/85RFZ+GSH+3vxjXA+0Xp5ql+U2JLwcwAv3dCYpOoiLlbU8M38PtTJ1jlP4cutpFuzMRaOB93+fQlSwn9oluRUJLweg9fJk9tge6Hy92JNzkXdWy6K1ju7H4+eZvuIIANOGJtC/c2uVK3I/El4OIjbMn5n3JQHwyeaT/PenIpUrEo05db6CJ7/ejcms8JseUdbTfmFfEl4OZEi3SB5Obw/A8wv3c6akQt2CxK9crKxh4pc7MVTVkRIbzNtjZJobtUh4OZhpwxJIjgmm7FItk77aJeO/HEhVrYnHvtrNyXMVRAb58slDqfh6Swe9WiS8HIzWy5NPH0qljU7LsSIjLyzah9msqF2W2zObFV5YtJ8dpy+g03ox75HbaaOT+bnUJOHlgML1vnz8UCo+nh6sPlTEh/89rnZJbu+tlUdYkVWAt6eGT8ankhChV7sktyfh5aB6xIbw5uX7H99fd5zlB86qXJH7+mTTCf7vj5aVz9/5XTLpHWWmCEcg4eXA7r8thj/26QDAlIX72X6yROWK3M8/t50h4/KiwS8PTWBUd7n1x1FIeDm4Pw/vwuCu4dSYzEz6ahfZxeVql+Q2vt2dx1+WHgTgyQEdebx/R5UrEleS8HJwnh4aPvh9Cj1igzFU1THh850UG2T5tJa2KquAqYv3A/Bwenvr8nXCcUh4OQFfb08+m3A7HVoFkH/xEuM/30FpRY3aZbms5QfO8vS/92JW4Hep0fzviEQZy+WAJLycRGiAD18+0pM2Oi0/FZbz0OfbKbtUq3ZZLmfJ3jye/fdeTGaFMSlRzPhtEh4eElyOSNXw+uijj0hKSkKv16PX60lLS2PVqlXXfM4333xDQkICvr6+3HrrraxcudJO1aovNsyf+ZN6ERbgw8F8Aw/P2yGDWG1o0a5cpizaj1mB+2+L5p3fJeMpweWwVA2v6OhoZsyYwe7du9m1axd33XUXo0aN4tChQ1c9fuvWrYwdO5aJEyeyd+9eRo8ezejRozl48KCdK1dPXBsd/5zYiyA/b/bmXOSP83ZKgNnA3M0neWnxARQFHuwdy4zfJElwOTiNoigONXw7NDSUmTNnMnHixF899sADD1BRUcHy5cut+3r37k337t35+OOPm/T6BoOBoKAgysrK0Oudd6DhgbyLjJu7nfLqOpJjgvnykdsJ9negJbYqKiAw0PK10QgBjjktstms8NbKI9ZxXBP7duDV4V2kj0tFTf2MOkyfl8lkYsGCBVRUVJCWlnbVYzIzMxk4cGCDfYMHDyYzM7PR162ursZgMDTYXEFSdDBfT+pFiL83+3Mv8sAn2ygul6uQN6K6zsRzC/dZg+uVYQkSXE5E9fDKysoiMDAQrVbL448/zpIlS0hMTLzqsYWFhYSHhzfYFx4eTmFh49MnZ2RkEBQUZN1iYmJsWr+akqKDWfg/abTRaTlaVM79H2eSU1KpdllO4byxmgc/285/9p/Fy0PDew8k81i/jhJcTkT18IqPj2ffvn1s376dJ554ggkTJnD4sO0Wo5g2bRplZWXWLTc312av7Qg6h+tY/Hg60SF+nC6pZMw/trD7jCyldi2HzpYxavYWdp4uRaf14vOHb2dMSrTaZYkbpHp4+fj4EBcXR2pqKhkZGSQnJ/PBBx9c9diIiAiKihpO0ldUVERERESjr6/Vaq1XM+s3VxMb5s+3T6TTLUpPSUUNY+duZ9m+fLXLckgrDhRw30eZ5F+8RIdWASx5qg/9ZBZUp6R6eP2S2Wymurr6qo+lpaWxfv36BvvWrl3baB+ZOwnX+7Lof9IYlBhOTZ2ZyQv28e6ao5hkOh3A0r/12rKDPDV/D5dqTdzRqRVLn+xDXJtAtUsTzeSl5ptPmzaNoUOHEhsbS3l5OfPnz2fjxo2sXr0agPHjxxMVFUVGRgYAkydPpn///syaNYvhw4ezYMECdu3axaeffqrmj+Ew/H28+PjBVGasOsLcH07x4X+z2Zt7kfcf6E5YoFbt8lRz+nwFT/97DwfzLRdr/qffLUwdHI+Xp8P93y1ugKrhVVxczPjx4ykoKCAoKIikpCRWr17NoEGDAMjJycHD4+d/YOnp6cyfP59XX32VV155hU6dOrF06VK6deum1o/gcDw9NPx5eCJdIvW8siSLH46fZ/iHPzJnXAqp7ULVLs+uFEVh0a5cpi8/grG6jhB/b969vzt3JrRRuzRhAw43zquluco4r6Y4WljOE19bpi329NDw1ICOPH1XJ3y87NDiUHmcV0HZJV7+NotNx84BcHv7ED4cm0JkkCxP5uicbpyXsL34CB3/ebovo7q3xWRW+PC/2Yyes4WfCl1jrNvVmM0K/96Rwz3vbWbTsXP4eHnwyrAEFjyWJsHlYqTl5SZWHCjg1aVZlFbW4u2p4bF+t/DUnXH4+7RQz4EKLa/9uRf532UH2Z9XBkByTDCzfpdEXBtdi7+3sJ2mfkYlvNxIcXkVr3x3kHVHLMNN2gb58pcRiQzpFmH7wZl2DK+Csku8v/Y4i3bnoiig03rx/KDOjE9rJ53yTkjCqxHuHF5g6cRefaiI6csPk3/xEgA9O4Ty4j3x9Oxgww59O4RXibGajzae4KttZ6ipMwPwm5QoXh6WICv7ODEJr0a4e3jVu1Rj4qNNJ/h40wnrB/+OTq2YMqgzKbEhN/8GLRheeaWVzNtymgU7cqioMQGWAP7TkHi3u6LqiiS8GiHh1dDZi5eYvSGbRTtzqbs8oDW1XQiP9GnP4K4ReDf3tMvG4aUoCntySvkq8wzLDxRYB992i9IzdXAC/Tq1kvsSXYSEVyMkvK4up6SSv//3OEv35VNrsvyTiAzyZUxKFKNTougcfoOd3jYKryJDFUv25rNoVy4nz1VY9/eJC2PSHbfQv3NrCS0XI+HVCAmvays2VPGv7TnM336G88af58nvEqlnWLcI+se3plvboOtPjdzM8FIUhePFRtYeLmLN4SL25160Pubn7cmwWyN5pE97ukUF3eiPJpyEhFcjJLyapqrWxNrDRSzbl8/Go+esp5RgmU8/rWMYydFBdIuybHpf74Yv0MTwulBRw/GicrLyy9h5+gK7TpdS8ovFRVLbhfC71GiGJ0Wi++X7CJcj4dUICa8bV1pRw/eHCtnwUzFbT5RcddrpVoE+RIf4Ex3iR6tALSFKDZNH9wBg3vdZVPr4UlNn5kJFDefKqzlnrOb0+YpfBRWAj5cH6R3DGJQYzsAu4YTr5cqhO5HwaoSE182pNZnZm3ORnacvkJVXRlZ+mXXIxZX8aqo48t59AHR5fjGXfBoPoJhQPzq30ZHaPoSe7UO5NToIrZdni/0MwrE19TOq6o3Zwvl4e3rQs0NogzFhZZW15JZWkldaSV7pJUora6i++PMtSCOT22L298fby4OwAB9a67S0DtQSFeJHXJvAlhvlL1yatLxEy3CSBTiE45Ebs4UQLk3CSwjhlCS8hBBOScJLCOGUJLyEEE5JwksI4ZQkvIQQTsntRgfWD2szGFx3HneHUPHzDBAYDGAyqVeLcCr1n83rDUF1u/AqLy8HICYmRuVK3EjbtmpXIJxQeXk5QUGNzx7idiPszWYzZ8+eRafTueU8UAaDgZiYGHJzc+UOAyfgjn8vRVEoLy+nbdu2DdZt/SW3a3l5eHgQHR2tdhmq0+v1bvNhcAXu9ve6VournnTYCyGckoSXEMIpSXi5Ga1Wy2uvvYZWq1W7FNEE8vdqnNt12AshXIO0vIQQTknCSwjhlCS8hBBOScJLCOGUJLyEEE5Jwks08N5779G1a1cSExN59tlnr3tzrLCvMWPGEBISwn333ddg//Lly4mPj6dTp0589tlnKlVnXzJUQlidO3eO3r17c+jQIby9venXrx/vvPMOaWlpapcmLtu4cSPl5eV8+eWXLF68GIC6ujoSExPZsGEDQUFBpKamsnXrVsLCwlSutmVJy0s0UFdXR1VVFbW1tdTW1tKmTRu1SxJXGDBgADqdrsG+HTt20LVrV6KioggMDGTo0KGsWbNGpQrtR8LLhWzevJl7772Xtm3botFoWLp06a+OmTNnDu3bt8fX15devXqxY8cO62OtW7fmxRdfJDY2lrZt2zJw4EA6duxox5/Atd3s36cxZ8+eJSoqyvp9VFQU+fn5tizdIUl4uZCKigqSk5OZM2fOVR9fuHAhU6ZM4bXXXmPPnj0kJyczePBgiouLASgtLWX58uWcPn2a/Px8tm7dyubNm+35I7i0m/37iIYkvFzI0KFDefPNNxkzZsxVH3/33XeZNGkSjzzyCImJiXz88cf4+/vz+eefA7Bu3Tri4uIIDQ3Fz8+P4cOHs23bNnv+CC7tZv8+jWnbtm2DllZ+fj5t3WACSAkvN1FTU8Pu3bsZOHCgdZ+HhwcDBw4kMzMTsMwuu3XrVqqqqjCZTGzcuJH4+Hi1SnYrTfn7NKZnz54cPHiQ/Px8jEYjq1atYvDgwS1dsurcbjJCd3X+/HlMJhPh4eEN9oeHh/PTTz8B0Lt3b4YNG0ZKSgoeHh7cfffdjBw5Uo1y3U5T/j4AAwcOZP/+/VRUVBAdHc0333xDWloas2bN4s4778RsNvPSSy+5/JVGkPASv/DWW2/x1ltvqV2GaMS6deuuun/kyJFu9x+NnDa6iVatWuHp6UlRUVGD/UVFRURERKhUlagnf58bJ+HlJnx8fEhNTWX9+vXWfWazmfXr18sgVAcgf58bJ6eNLsRoNJKdnW39/tSpU+zbt4/Q0FBiY2OZMmUKEyZM4LbbbqNnz568//77VFRU8Mgjj6hYtfuQv4+NKcJlbNiwQQF+tU2YMMF6zN///nclNjZW8fHxUXr27Kls27ZNvYLdjPx9bEvubRRCOCXp8xJCOCUJLyGEU5LwEkI4JQkvIYRTkvASQjglCS8hhFOS8BJCOCUJLyGEU5LwEkI4JQkv4dAGDBjAc8891+Tjv/jiC4KDg5v9fo3NLS8cj9yYLcQVCgoKCAkJUbsM0QQSXkJcQebOch5y2ihu2IABA3jmmWd47rnnCAkJITw8nLlz51qnb9HpdMTFxbFq1aoGz9u0aRM9e/ZEq9USGRnJyy+/TF1dnfXxiooKxo8fT2BgIJGRkcyaNetX711dXc2LL75IVFQUAQEB9OrVi40bNza59pqaGp5++mkiIyPx9fWlXbt2ZGRkWB+/8rTx9ddfR6PR/Gr74osvAMt8WxkZGXTo0AE/Pz+Sk5OtC8EKO1B7WgvhfPr376/odDpl+vTpyrFjx5Tp06crnp6eytChQ5VPP/1UOXbsmPLEE08oYWFhSkVFhaIoipKXl6f4+/srTz75pHLkyBFlyZIlSqtWrZTXXnvN+rpPPPGEEhsbq6xbt045cOCAMmLECEWn0ymTJ0+2HvPoo48q6enpyubNm5Xs7Gxl5syZilarVY4dO6YoiqLMmzdPCQoKarT2mTNnKjExMcrmzZuV06dPKz/88IMyf/586+OAsmTJEkVRFKW8vFwpKCiwbu+8847i7++vZGVlKYqiKG+++aaSkJCgfP/998qJEyeUefPmKVqtVtm4caNtftHimiS8xA3r37+/0rdvX+v3dXV1SkBAgPLQQw9Z9xUUFCiAkpmZqSiKorzyyitKfHy8YjabrcfMmTNHCQwMVEwmk1JeXq74+PgoixYtsj5eUlKi+Pn5WcPrzJkziqenp5Kfn9+gnrvvvluZNm2aoijXD69nnnlGueuuuxrUcaUrw+tKmZmZiq+vr7Jw4UJFURSlqqpK8ff3V7Zu3drguIkTJypjx45t9P2F7Uifl2iWpKQk69eenp6EhYVx6623WvfVr4JTv2DqkSNHSEtLQ6PRWI/p06cPRqORvLw8SktLqampoVevXtbHQ0NDGyy9lpWVhclkonPnzg1qqa6ubvJqOQ8//DCDBg0iPj6eIUOGMGLECO65555rPicnJ4fRo0fz4osvcv/99wOQnZ1NZWUlgwYNanBsTU0NKSkpTapF3BwJL9Es3t7eDb7XaDQN9tWHlNlsttl7Go1GPD092b17N56eng0eCwwMbNJr9OjRg1OnTrFq1SrWrVvH/fffz8CBAxvtq6qoqGDkyJGkpaXxxhtvNKgFYMWKFURFRTV4jlarvZEfSzSThJewiy5duvDtt9+iKIo12LZs2YJOpyM6OprQ0FC8vb3Zvn07sbGxAJSWlnLs2DH69+8PQEpKCiaTieLiYu64445m16LX63nggQd44IEHuO+++xgyZAgXLlwgNDS0wXGKovDggw9iNpv55z//2aDVmJiYiFarJScnx1qfsC8JL2EXTz75JO+//z7PPPMMTz/9NEePHuW1115jypQpeHh4EBgYyMSJE5k6dSphYWG0adOGP//5z3h4/HxBvHPnzowbN47x48cza9YsUlJSOHfuHOvXrycpKYnhw4dft453332XyMhI68K633zzDREREVcd2Pr666+zbt061qxZg9FotLa2goKC0Ol0vPjiizz//POYzWb69u1LWVkZW7ZsQa/XM2HCBJv97sTVSXgJu4iKimLlypVMnTqV5ORkQkNDmThxIq+++qr1mJkzZ2I0Grn33nvR6XS88MILlJWVNXidefPm8eabb/LCCy+Qn59Pq1at6N27NyNGjGhSHTqdjr/97W8cP34cT09Pbr/9dlauXNkgJOtt2rQJo9FIenr6r2p4+OGHmT59Oq1btyYjI4OTJ08SHBxMjx49eOWVV5rxGxI3ShbgEEI4JRmkKoRwShJeQginJOElhHBKEl5CCKck4SWEcEoSXkIIpyThJYRwShJeQginJOElhHBKEl5CCKck4SWEcEr/HzrztTTmmeQlAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -580,7 +537,8 @@ } ], "source": [ - "c = 1.92e19 # target compute budget (usually know this because we know how many GPU for how long go brrr)\n", + "c = 2.21e19 # target compute budget (usually know this because we know how many GPU for how long go brrr)\n", + "# (I got this flop number from row 1 of Table A3)\n", "# sweep model sizes from 10M to 100B\n", "ns = 10 ** np.arange(7, 11, step=2**-4)\n", "# using C = 6*N*D, solve for D that maintains the compute budget c\n", @@ -608,7 +566,7 @@ "source": [ "In the plot above, basically the models on the left of best are too small and trained for too long. The models on the right of best are way too large and trained for too little. The model at the red line is just right.\n", "\n", - "Now, the Chinchilla paper says that best model size is 400M params and 8B tokens, so this once again disagrees and there is some calculations problem. TODO figure out and fix..." + "Now, the Chinchilla paper says that best model size for this flop budget is 400M params and 9.2B tokens (instead of 316M params 11.65B params) so there is some unresolved disagreement here too..." ] }, { @@ -657,15 +615,15 @@ "output_type": "stream", "text": [ "closest model found:\n", - "model size: 1298.02M\n", - "dataset size: 60.32B\n", - "flops: 4.697589e+20\n", - "loss: 2.41\n" + "model size: 399.54M\n", + "dataset size: 14.43B\n", + "flops: 3.459892e+19\n", + "loss: 2.76\n" ] } ], "source": [ - "query_model_size = 1.3e9 # GPT-3 size\n", + "query_model_size = 400e6\n", "ns = np.array([n for c, n, d in models])\n", "ds = np.array([d for c, n, d in models])\n", "# find the index of the closest model size in ns\n", @@ -683,7 +641,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "So we predict 60B tokens is compute optimal. But e.g. [MosaicML quotes 26B](https://t.co/HyEvCqP70C). So again wrong." + "This should have come out as 9.2B according to Table A3 in Chinchilla paper, per my understanding of it." ] }, { @@ -691,7 +649,116 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "TLDR atm: nothing reproduces, but progress is being made." + "## Scaling Laws: Approach 2\n", + "\n", + "Approach 2 is probably my favorite one because it fixes a flop budget and runs a number of model/dataset sizes, measures the loss, fits a parabolla, and gets the minimum. So it's a fairly direct measurement of what we're after. The best way to then calculate the compute-optimal number of tokens for any given model size, as an example, is via simple interpolation." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# Approach 1 numbers\n", + "# # parameters, tokens\n", + "# raw = [\n", + "# [400e6, 8e9],\n", + "# [1e9, 20.2e9],\n", + "# [10e9, 205.1e9],\n", + "# [67e9, 1.5e12],\n", + "# [175e9, 3.7e12],\n", + "# [280e9, 5.9e12],\n", + "# [520e9, 11e12],\n", + "# [1e12, 21.2e12],\n", + "# [10e12, 216.2e12],\n", + "# ]\n", + "\n", + "# Approach 2 numbers\n", + "# parameters, tokens\n", + "raw = [\n", + " [400e6, 7.7e9],\n", + " [1e9, 20.0e9],\n", + " [10e9, 219.5e9],\n", + " [67e9, 1.7e12],\n", + " [175e9, 4.3e12],\n", + " [280e9, 7.1e12],\n", + " [520e9, 13.4e12],\n", + " [1e12, 26.5e12],\n", + " [10e12, 292.0e12],\n", + "]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y = 1.0409573169995892x + 0.9353887152390791\n" + ] + } + ], + "source": [ + "# fit a line by linear regression to the raw data\n", + "import numpy as np\n", + "x = np.array([np.log10(x[0]) for x in raw])\n", + "y = np.array([np.log10(x[1]) for x in raw])\n", + "A = np.vstack([x, np.ones(len(x))]).T\n", + "m, c = np.linalg.lstsq(A, y, rcond=None)[0]\n", + "print(f\"y = {m}x + {c}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATkAAAFBCAYAAAAMkNhdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCEElEQVR4nO3deVxU5f4H8M8MwozIJrHL5q6IgIIQWVdUFslISculEs2fdQ3SIm9JpYQt2oaWUmY3Nc2VNNyJRUwtlFwgvYpbaIoIIsLAINvM8/vjNKPDADIwcIaZ7/v18gXnmTNnvucBPp71OQLGGAMhhOgpId8FEEJIR6KQI4ToNQo5Qoheo5AjhOg1CjlCiF6jkCOE6DUKOUKIXqOQI4ToNQo5Qoheo5AjBisoKAhBQUEG99mamDlzJtzd3dv0Xl1ZRwo50iYff/wxUlJS+C7joc6dO4f3338fV69e5bsUwhMKOdImXSnkEhISmgy5tLQ0pKWldX5RpFN147sAQvhiYmLCdwmkE9CWXAcpLCzE7Nmz4eTkBJFIhN69e2Pu3Lmoq6tTzvPXX3/h2WefhbW1NUxNTfHoo49i3759Kss5dOgQBAIBtm/fjoSEBPTq1Qvm5uaYPHkyKioqUFtbi9dffx12dnYwMzPDrFmzUFtbq7IMgUCAmJgYbNq0CQMHDoRYLIavry8OHz6sMl9zx1/ef/99CAQCleVJpVL88MMPEAgEEAgEmDlzpsq6v/TSS7C3t4dIJMKQIUOwdu3aVvVbQ0MDPvjgA/Tt2xcikQju7u5455131NbJ3d0dTz31FNLS0uDj4wOxWAwPDw/s3LlTOc/69evx7LPPAgBGjx6trPXQoUMA1I8ZaaOv161bhzFjxsDOzg4ikQgeHh745ptvWrXuTVH87JKTk+Hh4YHu3bsjMDAQZ86cAQB8++236NevH8RiMYKCgprcYk1OToavry+6d+8OGxsbvPDCCygsLFSbLyUlBZ6enhCLxfD09MTPP//cZE1yuRwrVqzAkCFDIBaLYW9vj1deeQV379596PqsXLkSQ4YMgampKXr27Ak/Pz9s3rxZs07RFCNaV1hYyJycnJipqSl7/fXX2erVq9miRYvY4MGD2d27dxljjN26dYvZ29szc3Nz9u6777LExETm7e3NhEIh27lzp3JZWVlZDADz8fFhgYGB7KuvvmLz5s1jAoGATZ06lU2fPp2Fh4ezpKQk9uKLLzIALCEhQaUeAMzT05PZ2NiwJUuWsE8++YS5ubmx7t27szNnzijni4qKYm5ubmrrEx8fzx78Vdm4cSMTiUTsiSeeYBs3bmQbN25kv//+u3K9nJ2dmYuLC1uyZAn75ptv2NNPP80AsOXLlz+076KiohgANnnyZJaUlMRmzJjBALCJEyeqzOfm5sYGDBjArKys2MKFC1liYiIbOnQoEwqFLC0tjTHG2JUrV9i8efMYAPbOO+8oa7116xZjjLFRo0axUaNGabWvR4wYwWbOnMmWL1/OVq5cyUJDQxkAtmrVKpX5Gn92cwAwLy8v5uLiwpYtW8aWLVvGLC0tmaurK1u1ahXz8PBgX3zxBXvvvfeYiYkJGz16tMr7161bxwCwESNGsOXLl7OFCxey7t27M3d3d+XvImOM/fLLL0woFDJPT0+WmJjI3n33XWZpacmGDBmi9jvxf//3f6xbt25szpw5bPXq1eztt99mPXr0YCNGjGB1dXXNruOaNWuUP9tvv/2Wffnll2z27Nls3rx5D+2H9qCQ6wAzZsxgQqGQ/fHHH2qvyeVyxhhjr7/+OgPAjhw5onytsrKS9e7dm7m7uzOZTMYYu/+H5+npqfILNG3aNCYQCFh4eLjK8gMDA9V+KQEwAOzEiRPKtmvXrjGxWMwiIyOVba0NOcYY69GjB4uKilKbd/bs2czR0ZGVlpaqtE+dOpVZWlqy6upqtfco5ObmMgDs//7v/1TaFyxYwACwgwcPKtvc3NwYALZjxw5lW0VFBXN0dGTDhg1TtiUnJzMALCsrS+3zmgu59vR1U+sXFhbG+vTp0+JnNwcAE4lErKCgQNn27bffMgDMwcGBSSQSZXtcXBwDoJy3rq6O2dnZMU9PT3bv3j3lfHv37mUA2OLFi5VtPj4+zNHRkZWXlyvb0tLSGACVdTxy5AgDwDZt2qRSZ2pqqlp743WcMGECGzJkyEPXWdtod1XL5HI5UlJSEBERAT8/P7XXFbt9+/fvh7+/Px5//HHla2ZmZnj55Zdx9epVnDt3TuV9M2bMgLGxsXI6ICAAjDG89NJLKvMFBATg+vXraGhoUGkPDAyEr6+vctrV1RUTJkzAL7/8AplM1vYVfgBjDDt27EBERAQYYygtLVX+CwsLQ0VFBU6dOtXs+/fv3w8AiI2NVWl/8803AUBtV97JyQmRkZHKaQsLC8yYMQOnT5/GrVu32rwe7enr7t27K7+vqKhAaWkpRo0ahb/++gsVFRVtqmfs2LEqhxECAgIAAJMmTYK5ubla+19//QUAOHHiBEpKSvDqq69CLBYr5xs/fjwGDRqk7M+ioiLk5uYiKioKlpaWyvlCQkLg4eGhUktycjIsLS0REhKi8vP19fWFmZkZsrKyml0PKysr3LhxA3/88Ueb+qGtKOS07Pbt25BIJPD09GxxvmvXrmHgwIFq7YMHD1a+/iBXV1eVacUvo4uLi1q7XC5X+4Pq37+/2mcNGDAA1dXVuH37dou1ttbt27dRXl6ONWvWwNbWVuXfrFmzAAAlJSXNvv/atWsQCoXo16+fSruDgwOsrKzU+qRfv34qxwoV6wSgXZeMtKevf/vtNwQHB6NHjx6wsrKCra0t3nnnHQBoc8hpUg8A5bExRX819Xs2aNAg5euKr039jjR+76VLl1BRUQE7Ozu1n3FVVVWLP9+3334bZmZm8Pf3R//+/REdHY3ffvut+RXXEjq72kUYGRlp1M7aMKp948BQaO2WnlwuBwC88MILiIqKanIeLy+vNtfRWdra11euXMHYsWMxaNAgJCYmwsXFBSYmJti/fz+WL1+u7J/OqqcjyOVy2NnZYdOmTU2+bmtr2+x7Bw8ejAsXLmDv3r1ITU3Fjh078PXXX2Px4sVISEjoqJIp5LTN1tYWFhYWOHv2bIvzubm54cKFC2rt+fn5yte16dKlS2ptFy9ehKmpqfIXs2fPnigvL1ebr/EWFNB0ENna2sLc3BwymQzBwcEa1+jm5ga5XI5Lly4pt2gBoLi4GOXl5Wp9cvnyZTDGVGq5ePEiACh37zozMPfs2YPa2lrs3r1bZeurpV24jqTorwsXLmDMmDEqr124cEH5uuJrU78jjX9H+/bti4yMDIwcOVJl17y1evTogSlTpmDKlCmoq6vDM888g48++ghxcXEqu9TaRLurWiYUCjFx4kTs2bMHJ06cUHtd8b/sk08+iZycHGRnZytfk0qlWLNmDdzd3dWOhbRXdna2yvGw69evY9euXQgNDVVuEfTt2xcVFRX4888/lfMVFRU1eSlBjx491ALRyMgIkyZNwo4dO5oM+YftFj/55JMAgBUrVqi0JyYmAuCOJT3o5s2bKrVJJBJs2LABPj4+cHBwUNYJoMnw1jZFPz64JVVRUYF169Z1+Gc3xc/PD3Z2dli9erXKpS4HDhzA+fPnlf3p6OgIHx8f/PDDDyq71Onp6WrHhp977jnIZDJ88MEHap/X0NDQYj/fuXNHZdrExAQeHh5gjKG+vr4tq9gqtCXXAT7++GOkpaVh1KhRePnllzF48GAUFRUhOTkZR48ehZWVFRYuXIgtW7YgPDwc8+bNg7W1NX744QcUFBRgx44dEAq1+/+Pp6cnwsLCMG/ePIhEInz99dcAoLKbMHXqVLz99tuIjIzEvHnzUF1djW+++QYDBgxQO2Hg6+uLjIwMJCYmwsnJCb1790ZAQACWLVuGrKwsBAQEYM6cOfDw8EBZWRlOnTqFjIwMlJWVNVujt7c3oqKisGbNGpSXl2PUqFHIycnBDz/8gIkTJ2L06NEq8w8YMACzZ8/GH3/8AXt7e6xduxbFxcUqoeLj4wMjIyN88sknqKiogEgkUl7Hpm2hoaEwMTFBREQEXnnlFVRVVeG7776DnZ0dioqKtP55D2NsbIxPPvkEs2bNwqhRozBt2jQUFxfjyy+/hLu7O9544w3lvEuXLsX48ePx+OOP46WXXkJZWZnymraqqirlfKNGjcIrr7yCpUuXIjc3F6GhoTA2NsalS5eQnJyML7/8EpMnT26yntDQUDg4OGDkyJGwt7fH+fPnsWrVKowfP17lBIrWdfr5XANx7do1NmPGDGZra8tEIhHr06cPi46OZrW1tcp5rly5wiZPnsysrKyYWCxm/v7+bO/evSrLUVzWkJycrNKuuP6p8WUqiss9bt++rWwDwKKjo9mPP/7I+vfvz0QiERs2bFiTl1WkpaUxT09PZmJiwgYOHMh+/PHHJi8hyc/PZ//6179Y9+7dGQCVy0mKi4tZdHQ0c3FxYcbGxszBwYGNHTuWrVmz5qH9Vl9fzxISEljv3r2ZsbExc3FxYXFxcaympkZlPjc3NzZ+/Hj2yy+/MC8vLyYSidigQYPU+okxxr777jvWp08fZmRkpHI5SXOXkLSnr3fv3s28vLyYWCxm7u7u7JNPPmFr165VubSjqc9ujuJn96CCggIGgH322Wcq7c3Vv23bNjZs2DAmEomYtbU1e/7559mNGzfUPmvHjh1s8ODBTCQSMQ8PD7Zz585mLytas2YN8/X1Zd27d2fm5uZs6NCh7K233mI3b95sdh2//fZb9q9//Ys98sgjTCQSsb59+7L//Oc/rKKi4qH90B4Cxui5q/pOIBAgOjoaq1at4rsUrXF3d4enpyf27t3LdylEx9ExOUKIXqOQI4ToNQo5Qoheo2NyhBC9RltyhBC9RiFHCNFrdDFwM+RyOW7evAlzc3Pe76UkhKhijKGyshJOTk4PvXCeQq4ZN2/eVBvlgRCiW65fvw5nZ+cW56GQa4biNpPr16/DwsKC52q6jvr6eqSlpSlv9yGdy1D6XyKRwMXFpVW3g1HINUOxi2phYUEhp4H6+nqYmprCwsJCr//IdJWh9X9rDiXRiQdCiF6jkGskKSkJHh4eGDFiBN+lEEK0gEKukejoaJw7d67Tx6EnhHQMOiZHCNEJMjlDTkEZSiprYGcuhn9vaxgJ23/5FoUcIYR3qWeLkLDnHIoqapRtjpZixEd4YJynY7uWTburhBBepZ4twtwfT6kEHADcqqjB3B9PIfVs+0ZVppAjhPBGJmdI2HMODICAyRGXtRaDS7jnxipGDknYcw4yedvHEaGQa4TOrhLSeXIKylBUUQMBk+Pj1FV4JWcnNmxfjB611QC4oCuqqEFOQfPPBnkYCrlG6OwqIZ2npJILuGUHVmLan2mQCYT4cPRsSEWmavO1FZ14IITwxs7UGJ/u/wrPns2ATCDEG0+9id0eo9TnM2/7M1kp5Agh/JDJEPDRWwg8m4EGgRCvRyzA3sH/UplFAMDBkrucpK0o5AghnU8mA2bNgnDjRsiNjPD6+AXYN/gJlVkUV8jFR3i063o5OiZHCOlcMhkwcyawcSNgZAThli146qP5cLBU3SV1sBTjmxeGt/s6OdqSI4R0noYGICoK2LwZ6NYN2LoVmDQJ4wCEeDjQHQ+EkC6soQGYMQPYsoULuG3bgGeeUb5sJBQgsO8jWv9Y2l1thK6TI6QDNDQAL7xwP+CSk1UCriNRyDVC18kRomX19cD06dyWm7Ex8NNPwMSJnfbxtLtKCOk4ioD76Scu4HbsACIiOrUECjlCSMeorwemTeOCzcSE+/rUU51eBoUcIUT76uqAqVOBn3/mAm7nTmD8eF5KoZAjhGhXXR0wZQqQkgKIRFzQhYfzVg6FHCFEe+rqgGefBXbv5gIuJQUYN47XkijkCCHaUVvLBdyePYBYDOzaBYSG8l2VflxCEhkZiZ49e2Ly5Mlqr1VXV8PNzQ0LFizgoTJCDERtLTBp0v2A271bJwIO0JOQmz9/PjZs2NDkax999BEeffTRTq6IEANSU8Nd2LtvHxdwe/YAISF8V6WkFyEXFBQEc3NztfZLly4hPz8f4Twe9CREr9XUAJGRwP79QPfuwN69QHAw31Wp4D3kDh8+jIiICDg5OUEgECAlJUVtnqSkJLi7u0MsFiMgIAA5OTmtWvaCBQuwdOlSLVdMCAHABdzEiUBqKhdw+/YBY8fyXZUa3kNOKpXC29sbSUlJTb6+bds2xMbGIj4+HqdOnYK3tzfCwsJQUlLS4nJ37dqFAQMGYMCAAR1RNiGG7d49YMIE4JdfAFNTbktu9Gi+q2oS72dXw8PDW9ydTExMxJw5czBr1iwAwOrVq7Fv3z6sXbsWCxcubPZ9x44dw9atW5GcnIyqqirU19fDwsICixcvbnL+2tpa1NbWKqclEgkAoL6+HvX19W1ZNYOk6CvqM350Sv9XV8No0iQIMzPBevSAbPdusJEjuTscOokm68d7yLWkrq4OJ0+eRFxcnLJNKBQiODgY2dnZLb536dKlyl3V9evX4+zZs80GnGL+hIQEtfa0tDSYmpo28Q7SkvT0dL5LMGgd1f9GtbUI+Ogj2P75JxrEYmS/8w7KKiu5LblOVF1d3ep5dTrkSktLIZPJYG9vr9Jub2+P/Px85XRwcDDy8vIglUrh7OyM5ORkBAYGavRZcXFxiI2NVU5LJBK4uLggNDQUFhYW7VsRA1JfX4/09HSEhITA2NiY73IMTof2v1QKo8hICP/8E8zMDNizB4+OHKndz2glxZ5Wa+h0yLVWRkZGi6/PnDnzocsQiUQQiURISkpCUlISZDIZAMDY2Jj+WNuA+o1fWu9/qZQ7i3roEGBmBkFqKrrxFHAANFo33k88tMTGxgZGRkYoLi5WaS8uLoaDg0OHfCaNJ0dII1Ipd3P9oUOAuTl3soHHgNOUToeciYkJfH19kZmZqWyTy+XIzMzUeHeUENIGVVXAk08Cv/4KWFgAaWnAY4/xXZVGeN9draqqwuXLl5XTBQUFyM3NhbW1NVxdXREbG4uoqCj4+fnB398fK1asgFQqVZ5t1bbGu6uEGKzKSi7gjh69H3ABAXxXpTnGs6ysLAZA7V9UVJRynpUrVzJXV1dmYmLC/P392bFjxzq8roqKCgaAVVRUdPhn6ZO6ujqWkpLC6urq+C7FIGmt/yUSxkaOZAxgzNKSsePHtVKftmjy98n7llxQUBAYYy3OExMTg5iYmE6ph7bkiMGTSLjx337/HbC0BNLTgS78YCedPibHBzrxQAxaRQUQFsYFnJUVkJHRpQMO0IFjcoQQHaEIuOPHgZ49uYAbPpzvqtqNtuQaoeeuEoNUXs6N/3b8OGBtDWRm6kXAARRyamh3lRgcRcDl5NwPuGHD+K5KayjkCDFkd+9yA1z+8QfwyCPAwYOAjw/fVWkVHZMjxFCVlXEBd+oUYGPDbcF5efFdldbRllwjdEyOGISyMm4EX0XAHTyolwEHUMipoWNyRO/ducON4Hv6NGBrC2RlAUOH8l1Vh6GQI8SQlJZyAZebC9jZcQHn6cl3VR2KjskRYigUAffnn4C9PbeL6uHBd1UdjrbkGqFjckQv3b4NjBnDBZyDAzdskgEEHEAhp4aOyRG9U1LCBdyZM1zAZWUBgwbxXVWnod1VQvRZcTEXcOfOAY6OXMANHMh3VZ2KtuQI0Ve3bnGPCTx3DnBy4nZRDSzgAAo5QvSGTM6QU1AGADh9/DzY6NHA+fNAr15cwBnoM4hpd5UQPZB6tggJe86hrOoevuxXBpuXoiEoK8Q9e0d0P3QI6NeP7xJ5Q1tyjdDZVdLVpJ4twtwfT6GoogZ2lXfw+HvvoU9ZIQrNbTFuwhKk1vTgu0ReUcg1QmdXSVcikzMk7DkHBsC+shQbNr0Ls5s3UWhhi6nTl+Lvno5I2HMOMnnLo2/rMwo5QrqwnIIyFFXUwEFSiq1b4tC7rBDVtraY8cLHuG7lAAagqKJGeazOEFHIEdKFlVTWwFFymwu4u0W4YWmHox9+iBtWDmrzGSoKOUK6MOdKbgvOvbwI1y3t8eILS3HP3l5tPjtzMQ/V6QY6u0pIV3XtGoZHRUJQfgt/W9pj6vSluGNpC+D+k+YEABwsxfDvbc1bmXyjkCOkK7p2DQgKguDqVVQ7u2FqxPsosrCFCe6fYBD88zU+wgNGQkHTyzEAtLtKSFdz9SoQFMR97dsXptlHsfjVMDhYqu6SOliK8c0LwzHO05GPKnUGbck1Qg+XJjqtoIC7VevaNe4C30OHgF69MM4ZCPFwwLHLJSg9fwxro0bg0X52Br0Fp0Bbco3QdXJEZ/31F7cFd+0a0L+/MuAUjIQC5bE3/97WFHD/oJAjpCu4coULuL//5u5BbRRwpHm0u0qIjlHcaF9SWQM7czH8ZWUwGjsGuHGDG0UkK4sbNom0CoUcITpEcaN9UQV38a57WSG2b3sXdpJSbqDLrCxu4EvSahRyhOgIxY32iotAepcVYsuWONhVleHiI6648c02jKGA0xgdkyNEBzx4oz0A9LlzA1u3xMHhn4B7ftpHeDf7tkHfaN9WFHKE6ADFjfYA0PfOdWzZ+g7sq8pwwcYV06Z9jNs9ehr8jfZtpRchFxkZiZ49e2Ly5MnKtvLycvj5+cHHxweenp747rvveKyQkJYpbqDvW3odW7ZwAZdv44Zp05biTg8rtflI6+lFyM2fPx8bNmxQaTM3N8fhw4eRm5uL48eP4+OPP8adO3d4qpCQltmZi9Gv9G9s3RoHO+ldnLd1x/RpH6PM1FJtPqIZvQi5oKAgmJubq7QZGRnB1NQUAFBbWwvGGBij4xlEN/lXF2H71ndgKy3HObvemD71I5WAEwBwNPAb7duK95A7fPgwIiIi4OTkBIFAgJSUFLV5kpKS4O7uDrFYjICAAOTk5LRq2eXl5fD29oazszP+85//wMbGRsvVE6IFZ8/CaOwYWEvL8T+7Pnh+6ke42yjgALrRvq14DzmpVApvb28kJSU1+fq2bdsQGxuL+Ph4nDp1Ct7e3ggLC0NJSclDl21lZYW8vDwUFBRg8+bNKC4u1nb5hLTPmTPcvai3bwPDhuHWzj0QO9ipzEI32rcP79fJhYeHIzw8vNnXExMTMWfOHMyaNQsAsHr1auzbtw9r167FwoULW/UZ9vb28Pb2xpEjR1ROTjyotrYWtbW1ymmJRAIAqK+vR319fWtXx+Ap+or6rBX+/BPdxo2DoLQU8uHDIdu/H/+ytkaWXz+cvHYXpVW1sDETwdetJ4yEglb1qaH0vybrx3vItaSurg4nT55EXFycsk0oFCI4OBjZ2dktvre4uBimpqYwNzdHRUUFDh8+jLlz5zY7/9KlS5GQkKDWnpaWpjy2R1ovPT2d7xJ0mkVBAR5bvBiCykrc7dcP2bGxqD92TG2+UgC/nNd8+fre/9XV1a2eV6dDrrS0FDKZDPaNhnO2t7dHfn6+cjo4OBh5eXmQSqVwdnZGcnIyjIyM8PLLLytPOLz22msYOnRos58VFxeH2NhY5bREIoGLiwtCQ0NhYWGh/ZXTU/X19UhPT0dISAiMjY35Lkc35eai20svQVBZCbmfH8z270eIlZVWFm0o/a/Y02oNnQ651srIyGiyPTc3t9XLEIlEEIlEauPJGRsb6/UvS0ehfuOo3Wx/9yqMwsKAu3cBf38If/kFQi0F3IP0vf81WTedDjkbGxsYGRmpnTAoLi6GQwfdwxcdHY3o6GhIJBJYWlo+/A2ENKPxzfaety5j8/ZFsLhXCQQEAL/8AtDvWIfj/exqS0xMTODr64vMzExlm1wuR2ZmJgIDAzvkM5OSkuDh4YERI0Z0yPKJYXjwqfYAMLToEjZtfRcW9ypx0mkQ0ldsoIDrJLxvyVVVVeHy5cvK6YKCAuTm5sLa2hqurq6IjY1FVFQU/Pz84O/vjxUrVkAqlSrPtmobbcmR9mp8s71X0UX8uG0RLGqlONFrMGY9mwCzrOsY49+frnvrBLyH3IkTJzB69GjltOLgf1RUFNavX48pU6bg9u3bWLx4MW7dugUfHx+kpqaqnYwgRFc8eLO9980L2LhtESzqqvFHLw/MfPZ9SEWmqPznZvvAvo/wXK3+4z3kgoKCHnq7VUxMDGJiYjqlHnqQDWkvxU30PjcvYMM/AZfj7IFZk7mAazwf6Vg6fUyOD/QgG9JeduZiDC88j43b3oNFXTWOu3hi5rMJKgGnmI90PN635AjRN/5F+diYHI8edfdwzMUTL02OR7VJd+Xr9FT7zkVbco3Q2VXSLr/9BqPwcehRW41s16F4afL7agEH0M32nYlCrhHaXSVtdvQoMG4cUFUFjBmDyp9+hqWtlcosdLN956PdVULaQO1Ohhv/g9H4JwGpFBg7Fti9G6Gmphjr20d1Pnroc6ejkCNEQ43vZAj4+wzW7UiAaV0NEBIC7NoFdOd2UY2EArpMhGe0u9oIHZMjLWl8J8Ojf/+JdT+9D9O6Ghx2H4a0j1crA47oBgq5RuiYHGlO4zsZAq/9ibU/JcC0vha/9h6Ol595D/HpBfTYQB1DIUdIKz14J0PgtTxlwB3q7YuXn3kPNcYiemygDmp3yEkkEqSkpOD8+TaM7EdIF6K4Q2Hk1Vys+ykB3RtqcbCPH1555l3UdjNRm4/oBo1D7rnnnsOqVasAAPfu3YOfnx+ee+45eHl5YceOHVovsLPRMTnSHDtzMR4vOI3vdyyBuKEOmX1H4N+RqgGnmI/oDo1D7vDhw3jiiScAAD///DMYYygvL8dXX32FDz/8UOsFdjY6Jkea43/pBL7fyQVcej9/zJ34Duq63R+8kR4bqJs0DrmKigpYW3M/xNTUVEyaNAmmpqYYP348Ll26pPUCCdEJv/wCo4kTIGqoR1r/RxE9MU4t4AC6k0EXaRxyLi4uyM7OhlQqRWpqKkJDQwEAd+/ehVhMm+lEDx04AEyYANTWAhMngm3bikesVR9mTncy6C6NLwZ+/fXX8fzzz8PMzAxubm4ICgoCwO3GtvSgGEK6pP37gchIoK6O+7p1K8JMTBDs7Up3MnQRGofcq6++Cn9/f1y/fh0hISEQCrmNwT59+ujFMTlClPbuBSZN4gJu0iRgyxbgnweo0J0MXUebbuvy8/ODn5+fStv48eO1UhDfaNBMAgDYs4cLtvp6YPJkYPNmZcCRrkXjkJPJZFi/fj0yMzNRUlICuVyu8vrBgwe1Vhwf6BkPBLt2Ac8+ywXcc88BP/5IAdeFaRxy8+fPx/r16zF+/Hh4enpCIKDjEESPpKRwAdfQAEydCmzcCHSjcSy6Mo1/elu3bsX27dvx5JNPdkQ9hPDn55+5LbeGBmDaNGDDBgo4PaDxJSQmJibo169fR9RCCH927LgfcNOnU8DpEY1D7s0338SXX3750CdsEdJlJCcDU6ZwAffCCxRwekbjn+TRo0eRlZWFAwcOYMiQITBudEB2586dWiuOEG1RG8lXcV3b9u3clptMBrz4IrBuHWBkxHe5RIs0DjkrKytERkZ2RC2EdIjGI/kC3D2m33S7CJ+417iAi4oCvv+eAk4PaRxy69at64g6dAZdJ6dfFCP5Nj644p+diqF7EwEmB2bNAr77jgJOT7VpPLmGhgZkZGTg22+/RWVlJQDg5s2bqKqq0mpxfKBRSPRH45F8FSb8LwuJexNhxOTY7RcO2RoKOH2m8ZbctWvXMG7cOPz999+ora1FSEgIzM3N8cknn6C2tharV6/uiDoJ0diDI/kqTPxfFr7YtxxGTI4tXqF4Z8xc2F4rp1u09JjGW3Lz58+Hn58f7t69i+4PPLAjMjISmZmZWi2OkPZoPELvM2czlVtwm73H4Z1xMWACIY3kq+c03pI7cuQIfv/9d5iYqI6G6u7ujsLCQq0VRkh7PThC76Qzmfhs/woIwbDJZxzeC30VTCBUm4/oH41DTi6XN3lQ/saNGzA3N2/iHYTww7+3NRwtxXj8yB58cuArCMGwcdiTWBzybzCBEAJw48DRSL76TePd1dDQUKxYsUI5LRAIUFVVhfj4eLrVi+gUI6EAa+pzlQH3w/DxWBQyVxlwAI3kawg03pL74osvEBYWBg8PD9TU1GD69Om4dOkSbGxssGXLlo6okZC2+e47DI1fAADYHjgR8U/MBv4ZUMLBUoz4CA8aydcAaBxyzs7OyMvLw7Zt25CXl4eqqirMnj0bzz//vMqJiM4UGRmJQ4cOYezYsfjpp58AANevX8eLL76IkpISdOvWDYsWLcKzzz7LS32EB2vWAK+8wn0/fz4mfZEIl6t3aSRfQ8Q0tHnz5mZfW7BggaaL04qsrCy2e/duNmnSJGXbzZs32enTpxljjBUVFTEnJydWVVXV6mVWVFQwAKyiokLb5eq1uro6lpKSwurq6vgrYvVqxgDu3+uvMyaX81dLJ9OJ/u8Emvx9anxMbu7cuThw4IBa+xtvvIEff/xRC7GruaCgILWTHo6OjvDx8QEAODg4wMbGBmVl9GRzvffNN8C//819HxsLJCYqd1GJYdI45DZt2oRp06bh6NGjyrbXXnsN27dvR1ZWlsYFHD58GBEREXBycoJAIEBKSoraPElJSXB3d4dYLEZAQABycnI0+oyTJ09CJpPBxcVF4/pIF5KUBLz6Kvf9m28Cn39OAUc0D7nx48fj66+/xtNPP42TJ0/i1Vdfxc6dO5GVlYVBgwZpXIBUKoW3tzeSkpKafH3btm2IjY1FfHw8Tp06BW9vb4SFhaGkpKRVyy8rK8OMGTOwZs0ajWsjXcjKlUBMDPf9f/4DfPYZBRwB0MYH2UyfPh3l5eUYOXIkbG1t8euvv7Z5IM3w8HCEh4c3+3piYiLmzJmDWbNmAQBWr16Nffv2Ye3atVi4cGGLy66trcXEiROxcOFCPPbYYw+dt7a2VjktkUgAAPX19aivr2/t6hg8RV91Zp8JV66E0ZtvAgBkCxZA/uGH3NhwBoiP/ueDJuvXqpCLjY1tst3W1hbDhw/H119/rWxLTExs9Yc/TF1dHU6ePIm4uDhlm1AoRHBwMLKzs1t8L2MMM2fOxJgxY/Diiy8+9LOWLl2KhIQEtfa0tDSYmppqXryBS09P75TP6bN7N4auXQsAuDhpEs6PHMk9DNrAdVb/86W6urrV87Yq5E6fPt1ke79+/SCRSJSva/uhNqWlpZDJZLC3t1dpt7e3R35+vnI6ODgYeXl5kEqlcHZ2RnJyMmQyGbZt2wYvLy/lcb6NGzc2+wDsuLg4lTCXSCRwcXFBaGgoLCwstLpe+qy+vh7p6ekICQlRG1BV24QrVsDon4CTLVyI3gkJ6G3gu6id2f98UuxptUarQq4tJxQ6U0ZGRpPtjR+X2BKRSASRSKQ2npyxsbFe/7J0lA7vty++AN56i/t+0SIYJSTAyMAD7kH6/nurybq1aTw5hRs3buDGjRvtWUSLbGxsYGRkhOLiYpX24uJiODg4dMhn0nhyXcBnnwELuDsZEB8PLFlCJxlIszQOOblcjiVLlsDS0hJubm5wc3ODlZUVPvjgA422nFrDxMQEvr6+KkM4yeVyZGZmIjAwUKufpZCUlAQPDw+MGDGiQ5ZP2umTT+5vwb3/PvePkBZofHb13Xffxffff49ly5Zh5MiRALiH27z//vuoqanBRx99pNHyqqqqcPnyZeV0QUEBcnNzYW1tDVdXV8TGxiIqKgp+fn7w9/fHihUrIJVKlWdbtS06OhrR0dGQSCSwtLTskM8gbbR0KfDOO9z3CQnA4sX81kO6Bk1vp3B0dGS7du1Sa09JSWFOTk6aLo5lZWUxAGr/oqKilPOsXLmSubq6MhMTE+bv78+OHTum8edoim7rapsOu63oww/v36r1wQfaXbYeodu61Gm8JVdWVtbkRb+DBg1q021TQUFBD32Ga0xMDGIUF3p2MHqQjQ764IP7W20ffXR/a46QVtD4mJy3tzdWrVql1r5q1Sp4e3trpSg+0YkHHfPgbumDu6uEtJLGW3Kffvopxo8fj4yMDOXB/+zsbFy/fh379+/XeoHEgL3/PhdygOoJB0I0oPGWXO/evXHx4kVERkaivLwc5eXleOaZZ3DhwgW4ubl1RI2dis6u6gDGuEtDFAH32WcUcKTNNN6S6927N4qKitTOot65cwcuLi5d/lgWnV3lGWPc7umHH3LTn3/OjShCSBtpHHLNnSSoqqqCWExPPSLtwBjw3nvAxx9z04mJwBtv8FsT6fJaHXKK+zoFAgEWL16sctO6TCbD8ePHlYNUEqIxxriTCsuWcdMrVgDz5/NaEtEPrQ45xU34jDGcOXNG5bmrJiYm8Pb2xgLFrTZdGF1CwgPGgIULgU8/5aa/+gp47TV+ayJ6o9Uhp7hJf9asWfjyyy/1dmQOOibXyRjjTip8/jk3vWoVEB3Nb01Er2h8TG7dunUdUQcxRIxxo/h+8QU3/eDw5YRoSZtGBiak3RjjzpouX85Nf/01MHcuvzURvdSuoZb0EV0n1wkY486aKgJu9WoKONJhKOQaodu6Ohhj3FnTL7/kph98CDQhHYB2V0nnYYw7a5qUxA1y+d13wOzZfFdF9ByFHOkcjHGPDPz6ay7g/vtf4KWX+K6KGAAKOdLx5HIu4L75hgu4tWuBmTP5rooYCAo50rHkcu6ykG+/5QJu3TogKorvqogBoRMPjdDZVS2Sy4F///t+wP3wAwUc6XQUco3Q2dW2k8kZcgq40aFzrpRCPmcOd3JBKAQ2bABa8ZBvQrSNdleJVqSeLULCnnMoq7qHT/3kuDVlBoR/ZoAJhRBs3AhMn853icRAUciRdks9W4S5P54CA9BdIMOwVavg+udByARCvD7+TYz3Go1xfBdJDBbtrpJ2kckZEvacAwMglMvw8b6v4HrwIBoEQsyPWIC9HqOQsOccZPKWH1ZESEehkCPtklNQhqKKGgjlMny2fwUizxyEXCjEmxP/g72D/wUGoKiiRnmsjpDORrurpF1KKrmA+2LfckSeO4QGgRCnFixAarfHAZnqfITwgbbkSLvYde+GxH2JiDx3CPVCI7wR+RaKHntMfT5zGhqf8INCrhG6Tk4DDQ0IiH8dE8/9inqhEWImvI20QSNVZhEAcLQUw7+3NT81EoNHIdcIXSfXSg0NwPPPQ7htK+TdjBE9MQ5pA1S34AT/fI2P8ICRUKC+DEI6AYUc0Vx9PXfd2/btgLExhDt+wjMJr8LBUnWX1MFSjG9eGI5xno48FUoInXggmqqvB6ZNA3bsAExMuK9PPYVxAEI8HHDscglKzx/D2qgReLSfHW3BEd7Rlhxpvfp6YOrU+wG3cyfw1FPKl42EAuWxN//e1hRwRCfQlhxpnbo6LuB+/pkLuJ9/Bp58ku+qCHkoCjnycHV1wHPPAbt2ASIRkJICjKMbtUjXQCFHWlZbCzz7LLBnDxdwu3YBYWF8V0VIq+nFMbnIyEj07NkTkydPblU7aaXaWmDyZC7gxGJg924KONLl6EXIzZ8/Hxs2bGh1O2mF2lpg0iRg714u4PbsAUJD+a6KEI3pRcgFBQXB3Ny81e3kIWpqgGeeAfbtA7p354IuOJjvqghpE95D7vDhw4iIiICTkxMEAgFSUlLU5klKSoK7uzvEYjECAgKQk5PT+YUaipoaIDIS2L//fsCNHct3VYS0Ge8hJ5VK4e3tjaSkpCZf37ZtG2JjYxEfH49Tp07B29sbYWFhKCkp6eRKDcC9e8CECUBqKmBqygXdmDF8V0VIu/B+djU8PBzh4eHNvp6YmIg5c+Zg1qxZAIDVq1dj3759WLt2LRYuXKi1Ompra1FbW6uclkgkAID6+nrU19dr7XN01r17MJo0CcKMDDBTU8h27wYbOZK7AFgDir4yiD7TQYbS/5qsH+8h15K6ujqcPHkScXFxyjahUIjg4GBkZ2dr9bOWLl2KhIQEtfa0tDSYmppq9bN0jVFtLfw//hh2eXloEItx7N13caeqituSa6P09HQtVkg0pe/9X11d3ep5dTrkSktLIZPJYG9vr9Jub2+P/Px85XRwcDDy8vIglUrh7OyM5ORkBAYGNtvelLi4OMTGxiqnJRIJXFxcEBoaCgsLi45ZQV1QXQ2jyEgI8/LAzMyAPXsQMHLkw9/XjPr6eqSnpyMkJATGxsZaLJS0hqH0v2JPqzV0OuRaKyMjQ6P2pohEIohEIiQlJSEpKQkyGTesrbGxsf7+skil3EmGrCzAzAyC1FR0a0fAPUiv+60L0Pf+12TdeD/x0BIbGxsYGRmhuLhYpb24uBgODg4d8pkGM56cVMrdXJ+VBZibA7/8Amgp4AjRJTodciYmJvD19UVmZqayTS6XIzMzs9ndzvYyiJGBq6q4m+sPHbofcE0MWU6IPuB9d7WqqgqXL19WThcUFCA3NxfW1tZwdXVFbGwsoqKi4OfnB39/f6xYsQJSqVR5tlXboqOjER0dDYlEAktLyw75DF4pAu7IEcDCggu4Rx/luypCOgzvIXfixAmMHj1aOa04+B8VFYX169djypQpuH37NhYvXoxbt27Bx8cHqampaicjSCtUVnIBd/QoYGkJpKUB/v58V0VIh+I95IKCgsBYyw8ejomJQUxMTKfU0/jEg96QSIDwcOD337mAS08H9HmXnJB/6PQxOT7o5YkHiYQb/+333wErKyAjgwKOGAzet+RIB6uo4ALu2DGgZ08u4IYP57sqQjoNbck1oldnVysquPHfKOCIAaOQa0RvdlfLy7nx344fB6ytgcxMCjhikGh3VR/dvcsF3IkTwCOPcAHn7c13VYTwgrbkGunyu6t37wIhIVzA2dgABw9SwBGDRiHXSJfeXS0r40bwPXnyfsB5efFdFSG8ot1VfaEIuNOnAVtbLuA8PfmuihDe0ZacPrhzhxui/PRpwM6Ou+meAo4QABRyXV9pKRdwubmAvT0XcEOG8F0VITqDQq6RLnXi4fZt7hkMeXn3A87Dg++qCNEpFHKNdJkTDyUlXMCdOQM4OHDDJg0ezHdVhOgcOvHQFSkC7n//AxwduS24gQP5rooQnURbcl1NcTEwejQXcE5O3BYcBRwhzaKQ60pu3eIC7tw5oFcvLuAGDOC7KkJ0GoVcV1FUxAXc+fOAszMXcP37810VITqPQq4RnTy7qgi4/HzAxYULuH79+K6KkC6BQq4RnTu7evMmEBQEXLgAuLpyAde3L99VEdJlUMjpssJCLuAuXgTc3LiA69OH76oI6VLoEhIdIpMz5BSUoaSyBs5VZRg+6xkILl++H3Du7nyXSEiXQyGnI1LPFiFhzzkUVdTAUXIbW7a8A0F5Eap7ucD011+5oCOEaIx2V3VA6tkizP3xFIoqauAkKcHWLXFwLy/C35b2CHk6AamVJnyXSEiXRSHHM5mcIWHPOTCAC7jNcXArv4VrVg6YOn0pblrYIWHPOcjkLT+2kRDSNAo5nuUUlKGoogbOFcXYtjkOrhXFuGrliKnTuIBjAIoqapBTUMZ3qYR0SRRyjXT2dXIllVzAbd0cB5eKYhT05AKuyMJWbT5CiOYo5Brp7OvkXMqLsXXzQjhLSvBXTydMnbYUtyxs1OazMxd3Sj2E6Bs6u8qnv/7CsBkTIZDcxl/WvTB16scoMX9EZRYBAAdLMfx7W/NTIyFdHIUcX65cAUaPhuD6dVS598XU8fG4baYaZIJ/vsZHeMBIKFBfBiHkoWh3lQ+XL3N3Mly/DgwaBLPfj2DJv4PhYKm6S+pgKcY3LwzHOE9HfuokRA/Qllxnu3SJu9m+sJAbyffgQcDBAeMcgRAPB+UdD3bm3C4qbcER0j4Ucp3p4kUu4G7e5J7FcPAg92yGfxgJBQjs+0gLCyCEaEovdlcjIyPRs2dPTJ48WaV97969GDhwIPr374///ve/PFX3jwsXuF3Umze5p2llZakEHCGkY+hFyM2fPx8bNmxQaWtoaEBsbCwOHjyI06dP47PPPsOdO3f4KTA/nwu4oiJg6FAu4Ozs+KmFEAOjFyEXFBQEc3NzlbacnBwMGTIEvXr1gpmZGcLDw5GWltb5xZ0/zwXcrVtcwGVmck+4J4R0Ct5D7vDhw4iIiICTkxMEAgFSUlLU5klKSoK7uzvEYjECAgKQk5Pz0OXevHkTvXr1Uk736tULhYWF2iz94c6d4wKuuBjw8uKOwVHAEdKpeA85qVQKb29vJCUlNfn6tm3bEBsbi/j4eJw6dQre3t4ICwtDSUlJJ1eqof/9jzvJUFIC+PhwAWejficDIaRj8X52NTw8HOHh4c2+npiYiDlz5mDWrFkAgNWrV2Pfvn1Yu3YtFi5c2Oz7nJycVLbcCgsL4e/v3+z8tbW1qK2tVU5LJBIAQH19Perr61u9PgCAs2fRLSwMgtu3wXx80HDgAGBhAWi6nC5I0Vca9xnRCkPpf03Wj/eQa0ldXR1OnjyJuLg4ZZtQKERwcDCys7NbfK+/vz/Onj2LwsJCWFpa4sCBA1i0aFGz8y9duhQJCQlq7WlpaTA1NW11zeZXr2Lk4sUQSCQo79MHv7/5JuqPH2/1+/VFeno63yUYNH3v/+rq6lbPq9MhV1paCplMBvtGl1rY29sjPz9fOR0cHIy8vDxIpVI4OzsjOTkZgYGB+OKLLzB69GjI5XK89dZbeOSR5q9Bi4uLQ2xsrHJaIpHAxcUFoaGhsLCwaPI9MjnDyWt3UVpVCxszEfwqrsNk9mwIJBLIhw9HjwMHENKzZzt7oWupr69Heno6QkJCYGxszHc5BsdQ+l+xp9UaOh1yrZWRkdFk+9NPP42nn366VcsQiUQQiURq7cbGxk3+sjw4XDkAeBT/hc3b34OoWgL4+UGYlgahgQXcg5rrN9I59L3/NVk33k88tMTGxgZGRkYoLi5WaS8uLoaDg0OHfGZrxpN7cLhyABhSfAWbtr4Lq2oJ8hz7I+PLjYABBxwhukSnQ87ExAS+vr7IzMxUtsnlcmRmZiIwMLBDPvNh48k9OFw5AAy5dRmbtr6LnjWVOO04EC9O+RCLfi2k4coJ0RG8765WVVXh8uXLyumCggLk5ubC2toarq6uiI2NRVRUFPz8/ODv748VK1ZAKpUqz7ZqW1JSEpKSkiCTyZp8XTFcOQAI5TKs2PsFrGqqcMppIKKeW4JKUQ9I/hmunO5DJYR/vIfciRMnMHr0aOW04uB/VFQU1q9fjylTpuD27dtYvHgxbt26BR8fH6SmpqqdjNCW6OhoREdHQyKRwNLSUu31B4chlwuN8O+J7+Ctwz/gzfGxqBKZNjkfIYQ/vIdcUFAQGGt51y4mJgYxMTGdVFHLGg9DfsXGBa88895D5yOE8EOnj8nx4WEnHvx7W8PRUozmRnkTAHCk4coJ0RkUco087MSDkVCA+AgPAFALOhqunBDdQyHXBuM8HfHNC8NpuHJCugDej8npmoedXVUY5+lIw5UT0gVQyDXysLOrD6LhygnRfbS7SgjRaxRyjbTmti5CSNdBu6uNKHZXKyoqYGVlpdFoB4QbBaO6uhoSiUSvbxDXVYbS/4q/y4ddYwtQyDWrsrISAODi4sJzJYSQ5lRWVj702LmAtSYKDZBcLsfNmzdhbm4OgYDOmLaWYhy+69evNzsOH+k4htL/jDFUVlbCyckJQmHLR91oS64ZQqEQzs7OfJfRZVlYWOj1H5muM4T+f9gWnAKdeCCE6DUKOUKIXqOQI1olEokQHx/f5FDypONR/6ujEw+EEL1GW3KEEL1GIUcI0WsUcoQQvUYhRwjRaxRyhBC9RiFHOlRkZCR69uyJyZMnq7Tv3bsXAwcORP/+/fHf//6Xp+r0X1P9f/36dQQFBcHDwwNeXl5ITk7mscKOR5eQkA516NAhVFZW4ocffsBPP/0EAGhoaICHhweysrJgaWkJX19f/P7773jkERqAVNua6v+ioiIUFxfDx8cHt27dgq+vLy5evIgePXrwXG3HoC050qGCgoJgbm6u0paTk4MhQ4agV69eMDMzQ3h4ONLS0niqUL811f+Ojo7w8fEBADg4OMDGxgZlZWU8VNc5KORIsw4fPoyIiAg4OTlBIBAgJSVFbZ6kpCS4u7tDLBYjICAAOTk5D13uzZs30atXL+V0r169UFhYqM3S9UJH9f+DTp48CZlMptdDilHIkWZJpVJ4e3sjKSmpyde3bduG2NhYxMfH49SpU/D29kZYWBhKSko6uVL91NH9X1ZWhhkzZmDNmjXaLFv3MEJaAQD7+eefVdr8/f1ZdHS0clomkzEnJye2dOlSlfmysrLYpEmTlNO//fYbmzhxonJ6/vz5bNOmTR1TuJ7QZv8zxlhNTQ174okn2IYNGzqsZl1BW3KkTerq6nDy5EkEBwcr24RCIYKDg5Gdnd3ie/39/XH27FkUFhaiqqoKBw4cQFhYWEeXrFfa0/+MMcycORNjxozBiy++2NGl8o4GzSRtUlpaCplMBnt7e5V2e3t75OfnK6eDg4ORl5cHqVQKZ2dnJCcnIzAwEF988QVGjx4NuVyOt956i86saqg9/S+TybBt2zZ4eXkpj/Nt3LgRQ4cO7cxV6DQUcqRDZWRkNNn+9NNP4+mnn+7kagxPc/0vl8s7uRL+0O4qaRMbGxsYGRmhuLhYpb24uBgODg48VWU4qP9bj0KOtImJiQl8fX2RmZmpbJPL5cjMzERgYCCPlRkG6v/Wo91V0qyqqipcvnxZOV1QUIDc3FxYW1vD1dUVsbGxiIqKgp+fH/z9/bFixQpIpVLMmjWLx6r1B/W/lvB9epforqysLAZA7V9UVJRynpUrVzJXV1dmYmLC/P392bFjx/grWM9Q/2sH3btKCNFrdEyOEKLXKOQIIXqNQo4Qotco5Agheo1CjhCi1yjkCCF6jUKOEKLXKOQIIXqNQo4Qotco5Ah5CHd3d6xYsYLvMkgbUcgRncYYQ0NDA99laEVdXR3fJRgkCjmiVUFBQYiJiUFMTAwsLS1hY2ODRYsWQXGL9MaNG+Hn5wdzc3M4ODhg+vTpKg9eOXToEAQCAQ4cOABfX1+IRCIcPXoUV65cwYQJE2Bvbw8zMzOMGDFCbUBId3d3fPjhh5gxYwbMzMzg5uaG3bt34/bt25gwYQLMzMzg5eWFEydOqLzv6NGjeOKJJ9C9e3e4uLhg3rx5kEqlyvW5du0a3njjDQgEAggEgla9T1HPBx98gBkzZsDCwgIvv/wy6urqEBMTA0dHR4jFYri5uWHp0qVa/zmQB/A7PgDRN6NGjWJmZmZs/vz5LD8/n/3444/M1NSUrVmzhjHG2Pfff8/279/Prly5wrKzs1lgYCALDw9Xvl8x8oaXlxdLS0tjly9fZnfu3GG5ubls9erV7MyZM+zixYvsvffeY2KxmF27dk35Xjc3N2Ztbc1Wr17NLl68yObOncssLCzYuHHj2Pbt29mFCxfYxIkT2eDBg5lcLmeMMXb58mXWo0cPtnz5cnbx4kX222+/sWHDhrGZM2cyxhi7c+cOc3Z2ZkuWLGFFRUWsqKioVe9T1GNhYcE+//xzdvnyZXb58mX22WefMRcXF3b48GF29epVduTIEbZ58+YO/7kYMgo5olWjRo1SCRHGGHv77bfZ4MGDm5z/jz/+YABYZWUlY+x+yKWkpDz0s4YMGcJWrlypnHZzc2MvvPCCcrqoqIgBYIsWLVK2ZWdnMwDKsJo9ezZ7+eWXVZZ75MgRJhQK2b1795TLXb58uco8rX3fg08lY4yx1157jY0ZM0alf0jHot1VonWPPvqoym5dYGAgLl26BJlMhpMnTyIiIgKurq4wNzfHqFGjAAB///23yjL8/PxUpquqqrBgwQIMHjwYVlZWMDMzw/nz59Xe5+Xlpfxe8ZCXBx/QomhT7CLn5eVh/fr1MDMzU/4LCwuDXC5HQUFBs+vY2vc1Xo+ZM2ciNzcXAwcOxLx585CWltbsZxDtoJGBSaepqalBWFgYwsLCsGnTJtja2uLvv/9GWFiY2kH5Hj16qEwvWLAA6enp+Pzzz9GvXz90794dkydPVnufsbGx8ntF0DbVpniQS1VVFV555RXMmzdPrV5XV9dm16W172u8HsOHD0dBQQEOHDiAjIwMPPfccwgODsZPP/3U7GeR9qGQI1p3/Phxleljx46hf//+yM/Px507d7Bs2TK4uLgAgNpJgOb89ttvmDlzJiIjIwFwIXP16tV21zp8+HCcO3cO/fr1a3YeExMTyGQyjd/XHAsLC0yZMgVTpkzB5MmTMW7cOJSVlcHa2lrjZZGHo91VonV///03YmNjceHCBWzZsgUrV67E/Pnz4erqChMTE6xcuRJ//fUXdu/ejQ8++KBVy+zfvz927tyJ3Nxc5OXlYfr06Vp5rN7bb7+N33//HTExMcjNzcWlS5ewa9cuxMTEKOdxd3fH4cOHUVhYiNLS0la/rymJiYnYsmUL8vPzcfHiRSQnJ8PBwQFWVlbtXhfSNAo5onUzZszAvXv34O/vj+joaMyfPx8vv/wybG1tsX79eiQnJ8PDwwPLli3D559/3qplJiYmomfPnnjssccQERGBsLAwDB8+vN21enl54ddff8XFixfxxBNPYNiwYVi8eDGcnJyU8yxZsgRXr15F3759YWtr2+r3NcXc3Byffvop/Pz8MGLECFy9ehX79++HUEh/ih2FnvFAtCooKAg+Pj50hwDRGfTfByFEr1HIEUL0Gu2uEkL0Gm3JEUL0GoUcIUSvUcgRQvQahRwhRK9RyBFC9BqFHCFEr1HIEUL0GoUcIUSvUcgRQvTa/wM4VWbglJ1cKAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure(figsize=(3, 3))\n", + "# plot the line\n", + "plt.plot([q[0] for q in raw], [10**(m*np.log10(q[0]) + c) for q in raw], label='linear regression', color='r')\n", + "# plot the raw data\n", + "plt.scatter([q[0] for q in raw], [q[1] for q in raw], label='raw data')\n", + "plt.xscale('log')\n", + "plt.yscale('log')\n", + "plt.xlabel('parameters')\n", + "plt.ylabel('tokens')\n", + "plt.title('compute optimal models')\n", + "plt.grid()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "predicted parameters for 1.240000e+08 tokens: 2.292426e+09\n" + ] + } + ], + "source": [ + "xquery = 124e6 # query model size here (e.g. GPT-2 small is 124M)\n", + "yquery = 10**(m*np.log10(xquery) + c)\n", + "print(f\"predicted parameters for {xquery:e} tokens: {yquery:e}\")" ] } ],