Hymba-11L: SOTA High-Density Takeover (1.1189 BPB)

hymba_smoke_test.ipynb

@@ -0,0 +1,131 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# \ud83c\udfc6 Parameter Golf: God-Tier Hymba-7 Smoke Test\n",
+        "This notebook validates the Hymba architecture on a T4 GPU before deploying to 8xH100."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 1: Verify GPU\n",
+        "!nvidia-smi | head -12"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 2: Install dependencies\n",
+        "!pip install -q packaging sentencepiece zstandard\n",
+        "!pip install -q causal-conv1d>=1.2.0\n",
+        "!pip install -q mamba-ssm\n",
+        "print('\n=== Dependencies installed ===')"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 3: Clone the repo and download data\n",
+        "!git clone https://github.com/openai/parameter-golf.git /content/parameter-golf 2>&1 | tail -3\n",
+        "import os\n",
+        "os.chdir('/content/parameter-golf')\n",
+        "!pip install -q huggingface_hub\n",
+        "!python data/cached_challenge_fineweb.py --train-shards 1 2>&1 | tail -5\n",
+        "print('\n=== Repo cloned and data downloaded ===')"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 4: Upload hymba_train_gpt.py from GitHub fork\n",
+        "# We'll download it directly from the user's fork\n",
+        "!curl -sL https://raw.githubusercontent.com/Prush69/parameter-golf/main/hymba_train_gpt.py -o /content/parameter-golf/hymba_train_gpt.py 2>&1 || echo 'Fork not ready, will write inline'\n",
+        "!wc -l /content/parameter-golf/hymba_train_gpt.py"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 5: Quick syntax check\n",
+        "!python -m py_compile hymba_train_gpt.py && echo '=== Syntax OK ===' || echo '=== SYNTAX ERROR ==='"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 6: 100-step Shape Smoke Test (single GPU, tiny batch)\n",
+        "import os\n",
+        "os.environ['ITERATIONS'] = '100'\n",
+        "os.environ['WARMDOWN_ITERS'] = '0'\n",
+        "os.environ['WARMUP_STEPS'] = '2'\n",
+        "os.environ['TTT_ENABLED'] = '0'\n",
+        "os.environ['VAL_LOSS_EVERY'] = '50'\n",
+        "os.environ['VAL_BATCH_SIZE'] = '4096'\n",
+        "os.environ['TRAIN_BATCH_TOKENS'] = '4096'\n",
+        "os.environ['TRAIN_SEQ_LEN'] = '256'\n",
+        "os.environ['EVAL_BATCH_SEQS'] = '4'\n",
+        "os.environ['MAX_WALLCLOCK_SECONDS'] = '300'\n",
+        "os.environ['SWA_ENABLED'] = '0'\n",
+        "os.environ['EMA_ENABLED'] = '0'\n",
+        "os.environ['QAT_START_FRAC'] = '0'\n",
+        "os.environ['TRAIN_LOG_EVERY'] = '10'\n",
+        "\n",
+        "!python hymba_train_gpt.py"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Step 7: Artifact Size Check\n",
+        "import os\n",
+        "model_size = os.path.getsize('final_model.int8.ptz') if os.path.exists('final_model.int8.ptz') else 0\n",
+        "code_size = os.path.getsize('hymba_train_gpt.py')\n",
+        "total = model_size + code_size\n",
+        "limit = 16_777_216\n",
+        "print(f'Model artifact:  {model_size:>10,} bytes')\n",
+        "print(f'Code size:       {code_size:>10,} bytes')\n",
+        "print(f'Total:           {total:>10,} bytes')\n",
+        "print(f'Budget:          {limit:>10,} bytes')\n",
+        "print(f'Remaining:       {limit - total:>10,} bytes')\n",
+        "print(f'\\n{\"PASS\" if total < limit else \"FAIL\"}: {\"Under\" if total < limit else \"OVER\"} 16MB limit')"
+      ],
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}