osmarks/nanogpt-experiments
1
0
Fork 0
mirror of https://github.com/osmarks/nanogpt-experiments.git synced 2024-11-10 20:09:58 +00:00
Code Issues Releases Wiki Activity

Merge pull request #429 from adambala/fixes

Browse Source 
Open "shakespeare" data in UTF-8 in "prepare.py"
This commit is contained in:
Andrej 2024-02-27 09:05:44 -08:00 committed by GitHub
commit a022d02ee2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -8,3 +8,5 @@ __pycache__/
*.pt *.pt
*.pyc *.pyc
input.txt input.txt
env/
venv/

4
data/shakespeare/prepare.py
View File

@ -7,10 +7,10 @@ import numpy as np
input_file_path = os.path.join(os.path.dirname(__file__), 'input.txt') input_file_path = os.path.join(os.path.dirname(__file__), 'input.txt')
if not os.path.exists(input_file_path): if not os.path.exists(input_file_path):
data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt' data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'
with open(input_file_path, 'w') as f: with open(input_file_path, 'w', encoding='utf-8') as f:
f.write(requests.get(data_url).text) f.write(requests.get(data_url).text)
with open(input_file_path, 'r') as f: with open(input_file_path, 'r', encoding='utf-8') as f:
data = f.read() data = f.read()
n = len(data) n = len(data)
train_data = data[:int(n*0.9)] train_data = data[:int(n*0.9)]