results.json

{
  "experiment": "recurrent_ctx_interaction",
  "question": "Does recurrent depth amplify the benefit of longer context at tiny scale?",
  "timestamp": "2026-10-09T22:44:59Z",
  "config": {
    "d_model": 256,
    "n_heads": 4,
    "vocab": 256,
    "batch": 32,
    "steps": 2000,
    "lr": 0.0003,
    "device": "cuda"
  },
  "results": {
    "recurrent_ctx64": {
      "final_train_loss": 1.853339433670044,
      "avg_train_loss_last_100": 1.8315880990028381,
      "best_val_loss": 1.80549016436752,
      "final_val_loss": 1.80549016436752,
      "final_val_ppl": 6.08295236438963,
      "val_curve": [
        [
          200,
          2.67759696902061
        ],
        [
          400,
          2.4178282270626146
        ],
        [
          600,
          2.2530713908526363
        ],
        [
          800,
          2.111119897024972
        ],
        [
          1000,
          2.0204621748048432
        ],
        [
          1200,
          1.9230032847852123
        ],
        [
          1400,
          1.8634966042577004
        ],
        [
          1600,
          1.827961488645904
        ],
        [
          1800,
          1.8110538823263986
        ],
        [
          2000,
          1.80549016436752
        ]
      ],
      "params": 1397504,
      "train_time_s": 10.449508428573608
    },
    "recurrent_ctx256": {
      "final_train_loss": 2.2531955242156982,
      "avg_train_loss_last_100": 2.245353183746338,
      "best_val_loss": 2.235594819516552,
      "final_val_loss": 2.235594819516552,
      "final_val_ppl": 9.352042975781874,
      "val_curve": [
        [
          200,
          2.5640992836076384
        ],
        [
          400,
          2.4392180549855134
        ],
        [
          600,
          2.3669187321954843
        ],
        [
          800,
          2.323085509514322
        ],
        [
          1000,
          2.2937028310736833
        ],
        [
          1200,
          2.2744714941297257
        ],
        [
          1400,
          2.2546324447709685
        ],
        [
          1600,
          2.245326196904085
        ],
        [
          1800,
          2.2375385070333675
        ],
        [
          2000,
          2.235594819516552
        ]
      ],
      "params": 1446656,
      "train_time_s": 25.86048126220703
    },
    "stacked_ctx64": {
      "final_train_loss": 1.8972289562225342,
      "avg_train_loss_last_100": 1.9378939366340637,
      "best_val_loss": 1.9216734706139078,
      "final_val_loss": 1.9216734706139078,
      "final_val_ppl": 6.832382699238212,
      "val_curve": [
        [
          200,
          2.790465888198541
        ],
        [
          400,
          2.4131838263297567
        ],
        [
          600,
          2.278608127516143
        ],
        [
          800,
          2.14184259103269
        ],
        [
          1000,
          2.0658977236066547
        ],
        [
          1200,
          2.006630532595576
        ],
        [
          1400,
          1.9629625875122694
        ],
        [
          1600,
          1.9364350644909605
        ],
        [
          1800,
          1.9249796546235376
        ],
        [
          2000,
          1.9216734706139078
        ]
      ],
      "params": 1402880,
      "train_time_s": 7.927839279174805
    },
    "stacked_ctx256": {
      "final_train_loss": 2.2880020141601562,
      "avg_train_loss_last_100": 2.284539570808411,
      "best_val_loss": 2.2725286454570535,
      "final_val_loss": 2.2725286454570535,
      "final_val_ppl": 9.703907558344023,
      "val_curve": [
        [
          200,
          2.7255869145296057
        ],
        [
          400,
          2.4116121185069184
        ],
        [
          600,
          2.3523950440543038
        ],
        [
          800,
          2.3312676225389755
        ],
        [
          1000,
          2.3142523259532695
        ],
        [
          1200,
          2.298062850991074
        ],
        [
          1400,
          2.284544699532645
        ],
        [
          1600,
          2.278283355673965
        ],
        [
          1800,
          2.2737569964661892
        ],
        [
          2000,
          2.2725286454570535
        ]
      ],
      "params": 1452032,
      "train_time_s": 13.692972898483276
    }
  },
  "analysis": {
    "recurrent_ctx_gain": -3.2690906113922438,
    "stacked_ctx_gain": -2.8715248591058105,
    "recurrent_advantage_ctx64": 0.749430334848582,
    "recurrent_advantage_ctx256": 0.3518645825621487
  }
}