osmarks/nanogpt-experiments
1
0
Fork 0
mirror of https://github.com/osmarks/nanogpt-experiments.git synced 2024-09-21 03:39:44 +00:00
Code Issues Releases Wiki Activity

Merge pull request #270 from LaihoE/master

Browse Source 
fix np.sum overflows on windows
This commit is contained in:
Andrej 2023-06-14 15:36:26 -07:00 committed by GitHub
commit ed7887c888
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/openwebtext/prepare.py
View File

@ -50,7 +50,7 @@ tokenized = split_dataset.map(
# concatenate all the ids in each dataset into one large file we can use for training # concatenate all the ids in each dataset into one large file we can use for training
for split, dset in tokenized.items(): for split, dset in tokenized.items():
arr_len = np.sum(dset['len']) arr_len = np.sum(dset['len'], dtype=np.uint64)
filename = os.path.join(os.path.dirname(__file__), f'{split}.bin') filename = os.path.join(os.path.dirname(__file__), f'{split}.bin')
dtype = np.uint16 # (can do since enc.max_token_value == 50256 is < 2**16) dtype = np.uint16 # (can do since enc.max_token_value == 50256 is < 2**16)
arr = np.memmap(filename, dtype=dtype, mode='w+', shape=(arr_len,)) arr = np.memmap(filename, dtype=dtype, mode='w+', shape=(arr_len,))