mirror of
https://github.com/osmarks/nanogpt-experiments.git
synced 2024-09-21 03:39:44 +00:00
10 lines
248 B
Python
10 lines
248 B
Python
|
import numpy as np
|
||
|
import os
|
||
|
data_dir = "."
|
||
|
data = np.memmap(os.path.join(data_dir, 'train.bin'), dtype=np.uint16, mode='r')
|
||
|
datas = set(data)
|
||
|
vocab = set(range(50257))
|
||
|
unused = vocab - datas
|
||
|
unused = sorted(unused)
|
||
|
print(len(unused))
|
||
|
print(unused)
|