config.json

{
  "model_type": "tagforge",
  "architectures": [
    "TagForge"
  ],
  "vocab_size": 153728,
  "embed_dim": 1024,
  "n_layers": 16,
  "n_heads": 16,
  "dim_feedforward": 4096,
  "dropout": 0.0,
  "torch_dtype": "bfloat16",
  "checkpoint_step": 27800,
  "checkpoint_loss": 2.0304660201072693
}