PR openai#180 SOTA: 10L Int5-MLP + BigramHash(10240) + SWA(0.4) + WD=0.04

Reproduce openai/parameter-golf PR openai#180 (val_bpb 1.14276, 3-seed mean).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: joey00072

local.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+# PR #180 local 1-GPU test run
+# Reduces batch tokens to fit a single GPU (65536 vs 786432 for 8xH100)
+# GRAD_ACCUM logic: 786432 / 8 GPUs = 98304 tokens/step → approximate with 65536
+
+TRAIN_BATCH_TOKENS=65536 \
+TRAIN_SEQ_LEN=1024 \
+MAX_WALLCLOCK_SECONDS=120 \
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+torchrun --standalone --nproc_per_node=1 train_gpt.py

run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# PR #180 reproduction — 10L Int5-MLP + BigramHash(10240) + SWA(0.4) + WD=0.04
+# 3-seed mean val_bpb: 1.14276 | Best seed: 1.14260 (seed 2024)
+# All hyperparameters are defaults in train_gpt.py — no env vars needed.
+
+# Run with default seed (42):
+#   bash run.sh
+# Run with specific seed:
+#   SEED=1337 bash run.sh
+
+.venv/bin/torchrun --standalone --nproc_per_node=8 train_gpt.py