From 978d4fe538611a95c985cf04293121cba9b9167c Mon Sep 17 00:00:00 2001
From: Otavio Good <otaviogood@yahoo.com>
Date: Sat, 25 Mar 2023 00:04:45 -0700
Subject: [PATCH] Fix for gradient_accumulation_steps training slow

---
 config/train_gpt2.py             | 2 +-
 config/train_shakespeare_char.py | 1 +
 train.py                         | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/config/train_gpt2.py b/config/train_gpt2.py
index dab3d12..8f19273 100644
--- a/config/train_gpt2.py
+++ b/config/train_gpt2.py
@@ -10,7 +10,7 @@ wandb_run_name='gpt2-124M'
 # 12 batch size * 1024 block size * 5 gradaccum * 8 GPUs = 491,520
 batch_size = 12
 block_size = 1024
-gradient_accumulation_steps = 5
+gradient_accumulation_steps = 5 * 8
 
 # this makes total number of tokens be 300B
 max_iters = 600000
diff --git a/config/train_shakespeare_char.py b/config/train_shakespeare_char.py
index cb0d333..41c81df 100644
--- a/config/train_shakespeare_char.py
+++ b/config/train_shakespeare_char.py
@@ -14,6 +14,7 @@ wandb_project = 'shakespeare-char'
 wandb_run_name = 'mini-gpt'
 
 dataset = 'shakespeare_char'
+gradient_accumulation_steps = 1
 batch_size = 64
 block_size = 256 # context of up to 256 previous characters
 
diff --git a/train.py b/train.py
index 30d0145..1fe2b44 100644
--- a/train.py
+++ b/train.py
@@ -45,7 +45,7 @@ wandb_project = 'owt'
 wandb_run_name = 'gpt2' # 'run' + str(time.time())
 # data
 dataset = 'openwebtext'
-gradient_accumulation_steps = 5 # used to simulate larger batch sizes
+gradient_accumulation_steps = 5 * 8 # used to simulate larger batch sizes
 batch_size = 12 # if gradient_accumulation_steps > 1, this is the micro-batch size
 block_size = 1024
 # model
@@ -88,11 +88,12 @@ if ddp:
     torch.cuda.set_device(device)
     master_process = ddp_rank == 0 # this process will do logging, checkpointing etc.
     seed_offset = ddp_rank # each process gets a different seed
+    assert gradient_accumulation_steps % torch.cuda.device_count() == 0
+    gradient_accumulation_steps //= torch.cuda.device_count()
 else:
     # if not ddp, we are running on a single gpu, and one process
     master_process = True
     seed_offset = 0
-    gradient_accumulation_steps *= 8 # simulate 8 gpus
 
 if master_process:
     os.makedirs(out_dir, exist_ok=True)