From be571fff2cbba6bfb6b5c8ecf4fdfb0b8392b0a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E3=83=AA=E3=83=A7=E3=82=A6=E3=82=BC?=
 <98982999+ryouze@users.noreply.github.com>
Date: Mon, 16 Jan 2023 22:05:32 +0100
Subject: [PATCH] Improve readability of huge numbers

Before:
  length of dataset in characters:  1115394
  all the unique characters:
   !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
  vocab size: 65
  train has 1003854 tokens
  val has 111540 tokens

After:
  length of dataset in characters: 1,115,394
  all the unique characters:
   !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
  vocab size: 65
  train has 1,003,854 tokens
  val has 111,540 tokens
---
 data/shakespeare_char/prepare.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/data/shakespeare_char/prepare.py b/data/shakespeare_char/prepare.py
index 6759b2f..1a4e54d 100644
--- a/data/shakespeare_char/prepare.py
+++ b/data/shakespeare_char/prepare.py
@@ -17,13 +17,13 @@ if not os.path.exists('input.txt'):
 
 with open('input.txt', 'r') as f:
     data = f.read()
-print("length of dataset in characters: ", len(data))
+print(f"length of dataset in characters: {len(data):,}")
 
 # get all the unique characters that occur in this text
 chars = sorted(list(set(data)))
 vocab_size = len(chars)
 print("all the unique characters:", ''.join(chars))
-print("vocab size:", vocab_size)
+print(f"vocab size: {vocab_size:,}")
 
 # create a mapping from characters to integers
 stoi = { ch:i for i,ch in enumerate(chars) }
@@ -41,8 +41,8 @@ val_data = data[int(n*0.9):]
 # encode both to integers
 train_ids = encode(train_data)
 val_ids = encode(val_data)
-print(f"train has {len(train_ids)} tokens")
-print(f"val has {len(val_ids)} tokens")
+print(f"train has {len(train_ids):,} tokens")
+print(f"val has {len(val_ids):,} tokens")
 
 # export to bin files
 train_ids = np.array(train_ids, dtype=np.uint16)