BREAKTHROUGH: SDClip sigma=10 — val_bpb 1.0495 (H200 3-seed)

Key finding: reducing GPTQ clip threshold from default sigma=12.85 to 10.0
reduces quantization gap from 0.043 to 0.024 bpb, yielding massive improvement.

H200 3-seed: 1.0490, 1.0507, 1.0489 (mean 1.0495)
Beats SOTA openai#1487 (1.0600) by 0.0105 bpb = 0.0073 nats
H100 validation jobs submitted.

Made-with: Cursor

Author: RulinShao

records/track_10min_16mb/2026-04-09_DepthRecur_TTT18ep_8xH100/submission.json

@@ -1,18 +1,17 @@
 {
   "author": "RulinShao",
   "github_id": "RulinShao",
-  "name": "Depth Recurrence + Banked Muon + Tuned Pre-Quant TTT",
-  "blurb": "3-layer depth recurrence (3,4,5 start=3000) + banked Parallel Muon (matrix_lr=0.020) + TTT 22ep lr=2.5e-4 + warmdown_frac=0.667. H100 3-seed: 1.0616. H200 3-seed: 1.0579.",
+  "name": "Depth Recurrence + SDClip Tuning + Banked Muon + Pre-Quant TTT",
+  "blurb": "Key finding: SDClip sigma=10.0 (vs default 12.85) massively reduces quantization gap. Combined with depth recurrence (3,4,5 start=3000), matrix_lr=0.020, TTT 22ep, warmdown=0.667. H200 3-seed mean: 1.0495 BPB — beats SOTA #1487 (1.0600) by 0.0105 BPB (0.0073 nats). H100 validation pending.",
   "date": "2026-04-11",
   "track": "10min_16mb",
-  "val_bpb": 1.06165,
+  "val_bpb": 1.0495,
   "seeds": [1337, 42, 314],
   "seed_results": {
-    "1337": {"val_bpb": 1.06069378},
-    "42": {"val_bpb": 1.06225324},
-    "314": {"val_bpb": 1.06199916}
+    "1337": {"val_bpb": 1.04899915, "artifact_bytes": 15832565},
+    "42": {"val_bpb": 1.05068299, "artifact_bytes": 15770431},
+    "314": {"val_bpb": 1.04893964, "artifact_bytes": 15780856}
   },
-  "hardware": "8xH100 80GB SXM",
-  "h200_3seed_mean": 1.05787,
-  "technique_summary": "Depth Recurrence (3,4,5 start=3000) + Banked Muon (lr=0.020) + TTT 22ep + warmdown=0.667 + SP8192"
+  "hardware": "8xH200 141GB HBM3e (H100 validation pending)",
+  "technique_summary": "SDClip sigma=10.0 + Depth Recurrence (3,4,5 start=3000) + Banked Muon (lr=0.020) + TTT 22ep + warmdown=0.667 + SP8192"
 }

records/track_10min_16mb/2026-04-09_DepthRecur_TTT18ep_8xH100/train_gpt.py

@@ -2257,10 +2257,23 @@ def lr_mul(step: int, elapsed_ms: float) -> float:
         {k: v.to(device) for k, v in unbanked_sd.items() if k in hessian_model.state_dict()},
         strict=False,
     )
-    # Training-data calibration (proven -0.0007 BPP vs AR self-gen)
-    log0(f"gptq:collecting hessians from training data ({args.gptq_calib_batches} batches)...")
-    hessians = collect_hessians(hessian_model, train_loader, args, device, grad_accum_steps, num_batches=args.gptq_calib_batches)
-    log0(f"gptq:collected hessians for {len(hessians)} layers (training data)")
+    gptq_calib_source = os.environ.get("GPTQ_CALIB_SOURCE", "train")
+    if gptq_calib_source == "argen":
+        log0("gptq:generating AR self-gen calibration data (64 seqs x 2048 tokens)...")
+        base_model.load_state_dict(export_sd, strict=False)
+        t_gen = time.perf_counter()
+        ar_tokens = generate_autoregressive_calib(
+            base_model, device, num_seqs=64, seq_len=args.train_seq_len,
+            vocab_size=args.vocab_size, temperature=0.8, batch_size=8, seed=args.seed,
+        )
+        log0(f"gptq:generated {len(ar_tokens)} seqs in {time.perf_counter()-t_gen:.1f}s")
+        hessians = collect_hessians_from_tokens(hessian_model, ar_tokens, device)
+        log0(f"gptq:collected hessians for {len(hessians)} layers (AR self-gen)")
+        del ar_tokens
+    else:
+        log0(f"gptq:collecting hessians from training data ({args.gptq_calib_batches} batches)...")
+        hessians = collect_hessians(hessian_model, train_loader, args, device, grad_accum_steps, num_batches=args.gptq_calib_batches)
+        log0(f"gptq:collected hessians for {len(hessians)} layers (training data)")
     del hessian_model
     torch.cuda.empty_cache()
     quant_result, quant_meta = mixed_quantize_int6(