osmarks/nanogpt-experiments
1
0
Fork 0
mirror of https://github.com/osmarks/nanogpt-experiments.git synced 2024-12-18 14:10:28 +00:00
Code Issues Releases Wiki Activity

nanogpt: fix multiprocessing in load_dataset on os x

Browse Source 
The issue seems to be that _fixup_main_from_path in multiprocessing
module in python is unable to find entry point, thus, adding
```
if __name__ == '__main__'
```
This commit is contained in:
Oleksandr Kuvshynov 2023-06-17 20:35:38 -04:00
parent bb7e96754a
commit 542ac51d1f
1 changed files with 54 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
data/openwebtext/prepare.py
View File

@ -16,45 +16,46 @@ num_proc = 8
# it is better than 1 usually though # it is better than 1 usually though
num_proc_load_dataset = num_proc num_proc_load_dataset = num_proc
# takes 54GB in huggingface .cache dir, about 8M documents (8,013,769) if __name__ == '__main__':
dataset = load_dataset("openwebtext", num_proc=num_proc_load_dataset) # takes 54GB in huggingface .cache dir, about 8M documents (8,013,769)
dataset = load_dataset("openwebtext", num_proc=num_proc_load_dataset)
# owt by default only contains the 'train' split, so create a test split # owt by default only contains the 'train' split, so create a test split
split_dataset = dataset["train"].train_test_split(test_size=0.0005, seed=2357, shuffle=True) split_dataset = dataset["train"].train_test_split(test_size=0.0005, seed=2357, shuffle=True)
split_dataset['val'] = split_dataset.pop('test') # rename the test split to val split_dataset['val'] = split_dataset.pop('test') # rename the test split to val
# this results in: # this results in:
# >>> split_dataset # >>> split_dataset
# DatasetDict({ # DatasetDict({
# train: Dataset({ # train: Dataset({
# features: ['text'], # features: ['text'],
# num_rows: 8009762 # num_rows: 8009762
# }) # })
# val: Dataset({ # val: Dataset({
# features: ['text'], # features: ['text'],
# num_rows: 4007 # num_rows: 4007
# }) # })
# }) # })
# we now want to tokenize the dataset. first define the encoding function (gpt2 bpe) # we now want to tokenize the dataset. first define the encoding function (gpt2 bpe)
enc = tiktoken.get_encoding("gpt2") enc = tiktoken.get_encoding("gpt2")
def process(example): def process(example):
ids = enc.encode_ordinary(example['text']) # encode_ordinary ignores any special tokens ids = enc.encode_ordinary(example['text']) # encode_ordinary ignores any special tokens
ids.append(enc.eot_token) # add the end of text token, e.g. 50256 for gpt2 bpe ids.append(enc.eot_token) # add the end of text token, e.g. 50256 for gpt2 bpe
# note: I think eot should be prepended not appended... hmm. it's called "eot" though... # note: I think eot should be prepended not appended... hmm. it's called "eot" though...
out = {'ids': ids, 'len': len(ids)} out = {'ids': ids, 'len': len(ids)}
return out return out
# tokenize the dataset # tokenize the dataset
tokenized = split_dataset.map( tokenized = split_dataset.map(
process, process,
remove_columns=['text'], remove_columns=['text'],
desc="tokenizing the splits", desc="tokenizing the splits",
num_proc=num_proc, num_proc=num_proc,
) )
# concatenate all the ids in each dataset into one large file we can use for training # concatenate all the ids in each dataset into one large file we can use for training
for split, dset in tokenized.items(): for split, dset in tokenized.items():
arr_len = np.sum(dset['len'], dtype=np.uint64) arr_len = np.sum(dset['len'], dtype=np.uint64)
filename = os.path.join(os.path.dirname(__file__), f'{split}.bin') filename = os.path.join(os.path.dirname(__file__), f'{split}.bin')
dtype = np.uint16 # (can do since enc.max_token_value == 50256 is < 2**16) dtype = np.uint16 # (can do since enc.max_token_value == 50256 is < 2**16)
@ -71,9 +72,9 @@ for split, dset in tokenized.items():
idx += len(arr_batch) idx += len(arr_batch)
arr.flush() arr.flush()
# train.bin is ~17GB, val.bin ~8.5MB # train.bin is ~17GB, val.bin ~8.5MB
# train has ~9B tokens (9,035,582,198) # train has ~9B tokens (9,035,582,198)
# val has ~4M tokens (4,434,897) # val has ~4M tokens (4,434,897)
# to read the bin files later, e.g. with numpy: # to read the bin files later, e.g. with numpy:
# m = np.memmap('train.bin', dtype=np.uint16, mode='r') # m = np.memmap('train.bin', dtype=np.uint16, mode='r')