osmarks/nanogpt-experiments
1
0
Fork 0
mirror of https://github.com/osmarks/nanogpt-experiments.git synced 2024-09-21 03:39:44 +00:00
Code Issues Releases Wiki Activity

Merge pull request #82 from danielgross/master

Browse Source 
Missed two spots while relative pathing
This commit is contained in:
Andrej 2023-01-22 13:47:32 -08:00 committed by GitHub
commit 6c40a08b41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
data/shakespeare/prepare.py
View File

@ -4,12 +4,13 @@ import tiktoken
import numpy as np
# download the tiny shakespeare dataset
if not os.path.exists('input.txt'):
input_file_path = os.path.join(os.path.dirname(__file__), 'input.txt')
if not os.path.exists(input_file_path):
data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'
with open('input.txt', 'w') as f:
with open(input_file_path, 'w') as f:
f.write(requests.get(data_url).text)
with open('input.txt', 'r') as f:
with open(input_file_path, 'r') as f:
data = f.read()
n = len(data)
train_data = data[:int(n*0.9)]

7
data/shakespeare_char/prepare.py
View File

@ -10,12 +10,13 @@ import requests
import numpy as np
# download the tiny shakespeare dataset
if not os.path.exists('input.txt'):
input_file_path = os.path.join(os.path.dirname(__file__), 'input.txt')
if not os.path.exists(input_file_path):
data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'
with open('input.txt', 'w') as f:
with open(input_file_path, 'w') as f:
f.write(requests.get(data_url).text)
with open('input.txt', 'r') as f:
with open(input_file_path, 'r') as f:
data = f.read()
print(f"length of dataset in characters: {len(data):,}")