osmarks/nanogpt-experiments
1
0
Fork 0
mirror of https://github.com/osmarks/nanogpt-experiments.git synced 2024-12-18 14:10:28 +00:00
Code Issues Releases Wiki Activity

for consistency with previous commit

Browse Source
This commit is contained in:
Andrej Karpathy 2023-01-19 23:10:51 +00:00
parent c1c20a0311
commit 2c7806db6e
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
data/shakespeare/prepare.py
View File

@ -19,8 +19,8 @@ val_data = data[int(n*0.9):]
enc = tiktoken.get_encoding("gpt2") enc = tiktoken.get_encoding("gpt2")
train_ids = enc.encode_ordinary(train_data) train_ids = enc.encode_ordinary(train_data)
val_ids = enc.encode_ordinary(val_data) val_ids = enc.encode_ordinary(val_data)
print(f"train has {len(train_ids)} tokens") print(f"train has {len(train_ids):,} tokens")
print(f"val has {len(val_ids)} tokens") print(f"val has {len(val_ids):,} tokens")
# export to bin files # export to bin files
train_ids = np.array(train_ids, dtype=np.uint16) train_ids = np.array(train_ids, dtype=np.uint16)