osmarks/nanogpt-experiments
1
0
Fork 0
mirror of https://github.com/osmarks/nanogpt-experiments.git synced 2024-12-18 14:10:28 +00:00
Code Issues Releases Wiki Activity

reverse the order, making sure that the final layer init is preserved, and becomes the token embedding instead of the other way around. otherwise the loss can be all messed up from a bad init

Browse Source
This commit is contained in:
Andrej Karpathy 2023-01-14 02:16:10 +00:00
parent 7c8288552b
commit 43b37fd568
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
model.py
View File

@ -115,7 +115,7 @@ class GPT(nn.Module):
ln_f = nn.LayerNorm(config.n_embd), ln_f = nn.LayerNorm(config.n_embd),
)) ))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.lm_head.weight = self.transformer.wte.weight # https://paperswithcode.com/method/weight-tying self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
# report number of parameters # report number of parameters
n_params = sum(p.numel() for p in self.parameters()) n_params = sum(p.numel() for p in self.parameters())