{ "skill_version": "v0.4.0", "taxonomy": { "compute_structure": { "name": "compute structure", "values": [ { "value": "dense", "rule": "Every transformer layer's FFN is fully active per token; no routing anywhere.", "anchor": "Llama 3 (report describes no MoE; absence rule → inferred)" }, { "value": "sparse-MoE", "rule": "MoE layers dominate (≥80% of transformer layers are routed FFNs).", "anchor": "Mixtral 8x7B" }, { "value": "hybrid", "rule": "Neither pattern reaches 80% of layers (e.g. a substantial mix of dense and MoE layers).", "anchor": "None in atlas yet — first qualifying report sets the anchor" } ] }, "sequence_mixing": { "name": "sequence mixing", "values": [ { "value": "full-attention", "rule": "All layers use unrestricted attention over the whole context.", "anchor": "Llama 3" }, { "value": "efficient-attention", "rule": "Attention with restricted/computed patterns (sliding window, linear attention, sparse patterns) — but still attention.", "anchor": "Mistral (sliding window layers)" }, { "value": "SSM", "rule": "State-space layers (Mamba-style) do the mixing; no attention in the block.", "anchor": "Mamba" }, { "value": "hybrid", "rule": "Mixed attention + SSM/linear layers (or attention variants in a deliberate per-layer pattern).", "anchor": "Jamba (1 attention : 7 Mamba layers)" } ] }, "modality": { "name": "modality", "values": [ { "value": "text-only", "rule": "Report describes no non-text input or output.", "anchor": "Llama 3.1" }, { "value": "multimodal", "rule": "Report describes vision/audio/other modalities in or out.", "anchor": "GPT-4V (if its report is ever in the atlas — else first multimodal report ingested)" } ] }, "openness": { "name": "openness", "values": [ { "value": "open-weights-open-data", "rule": "Weights **and** training data released.", "anchor": "OLMo" }, { "value": "open-weights", "rule": "Weights released; data not released.", "anchor": "Llama 3 (report states weights release; data composition disclosed but not released)" }, { "value": "open-data", "rule": "Data released; weights not.", "anchor": "(rare — first qualifying report sets the anchor)" }, { "value": "closed", "rule": "Report says the model is not released.", "anchor": "GPT-4 (if its report is ever in the atlas)" }, { "value": "undisclosed", "rule": "Report is silent on release. Absence is itself the classification.", "anchor": "First qualifying report sets the anchor" } ] }, "scale_class": { "name": "scale class", "values": [ { "value": "frontier", "rule": "≥ 100B", "anchor": "Llama 3.1 405B" }, { "value": "large", "rule": "10B ≤ p < 100B", "anchor": "Llama 3.1 70B" }, { "value": "medium", "rule": "1B ≤ p < 10B", "anchor": "Llama 3.1 8B" }, { "value": "small", "rule": "< 1B", "anchor": "Qwen2-0.5B (if ingested)" } ] }, "design_intent": { "name": "design intent", "values": [ { "value": "frontier-generalist", "rule": "Framed as a general-purpose foundation model; broad benchmark coverage.", "anchor": "Llama 3.1" }, { "value": "reasoning-specialised", "rule": "Framed around reasoning/thinking (CoT, RLVR, test-time compute).", "anchor": "o1 (if its report is ever in the atlas)" }, { "value": "on-device", "rule": "Framed for edge/deployment efficiency constraints.", "anchor": "Phi-3-mini" }, { "value": "long-context", "rule": "Framed primarily around context length.", "anchor": "Gemini 1.5 (if in atlas)" }, { "value": "domain-specific", "rule": "Framed for a specific domain (code, biology, law…).", "anchor": "DeepSeek-Coder (if in atlas)" } ] } }, "labels": { "identity.family": "Family", "identity.variants": "Variants", "identity.org": "Organisation", "identity.release_date": "Release date", "identity.report_url": "Report URL", "identity.license": "License", "identity.open_weights": "Open weights", "scale.total_params": "Total parameters", "scale.active_params_per_token": "Active parameters / token", "scale.layers": "Layers", "scale.hidden_dim": "Hidden dimension", "scale.ffn_inner_dim": "FFN inner dimension", "scale.ffn_ratio": "FFN ratio", "scale.attention_heads_q": "Query heads", "scale.attention_heads_kv": "KV heads", "scale.head_dim": "Head dimension", "scale.vocab_size": "Vocabulary size", "scale.embedding_tied": "Embedding tying", "block.block_type": "Block type", "block.attention_variant": "Attention variant", "block.attention_layer_pattern": "Attention layer pattern", "block.depth_mixing": "Depth mixing", "block.activation": "Activation", "block.normalization.type": "Normalisation · Type", "block.normalization.placement": "Normalisation · Placement", "block.normalization.qk_norm": "Normalisation · QK-norm", "block.position_encoding.method": "Position encoding · Method", "block.position_encoding.rope_base": "Position encoding · RoPE base frequency", "block.position_encoding.partial_rope": "Position encoding · Partial RoPE", "block.position_encoding.extension": "Position encoding · Extension", "block.position_encoding.extension.method": "Position encoding · extension Method", "block.position_encoding.extension.factor": "Position encoding · extension Factor", "block.position_encoding.extension.target_length": "Position encoding · extension Target length", "block.stability.attention_sinks": "Stability · Attention sinks", "block.stability.softcapping": "Stability · Softcapping", "block.stability.other": "Stability · Other", "context.trained_length": "Trained context length", "context.deployed_length": "Deployed context length", "context.extension_method": "Extension method", "tokenizer.algorithm": "Algorithm", "tokenizer.notes": "Notes", "training.tokens": "Training tokens", "training.data_composition": "Data composition", "training.curriculum": "Curriculum / staging", "training.optimizer": "Optimizer", "training.lr_schedule": "LR schedule", "training.batch_schedule": "Batch schedule", "training.precision": "Precision", "training.parallelism": "Parallelism", "training.hardware": "Hardware", "training.compute": "Disclosed compute", "post_training.sft": "SFT", "post_training.preference_optimization": "Preference optimisation", "post_training.reasoning_training": "Reasoning training", "post_training.distillation": "Distillation", "modality.type": "Type", "modality.attachment": "Attachment", "efficiency.kv_cache": "KV-cache design", "efficiency.quantization": "Quantisation shipped", "efficiency.speculative_dedup": "Speculative decoding", "efficiency.serving": "Serving optimisations", "taxonomy.compute_structure": "Compute structure", "taxonomy.sequence_mixing": "Sequence mixing", "taxonomy.modality": "Modality", "taxonomy.openness": "Openness", "taxonomy.scale_class": "Scale class", "taxonomy.design_intent": "Design intent", "block.moe.expert_count": "MoE · Expert count", "block.moe.experts_per_token": "MoE · Experts per token", "block.moe.shared_experts": "MoE · Shared experts", "block.moe.routing": "MoE · Routing", "block.moe.load_balancing": "MoE · Load balancing", "block.moe.expert_granularity": "MoE · Expert granularity" }, "records": [ { "slug": { "value": "ai21-jamba", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.1", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2403.19887", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/ai21-jamba", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Jamba", "provenance": "stated", "ref": "Title / §1", "note": "Report: 'Jamba: A Hybrid Transformer-Mamba Language Model'; the model is called Jamba throughout." }, "variants": [], "org": { "value": "AI21 Labs", "provenance": "inferred", "note": "Report text never names the organisation; the model URL huggingface.co/ai21labs/Jamba-v0.1 (§1, Abstract) identifies it as AI21 Labs." }, "release_date": { "value": "2024-03-28", "provenance": "inferred", "note": "arXiv v1 submission date (ID from report URL; arXiv API metadata) — report text itself is undated" }, "report_url": { "value": "https://arxiv.org/abs/2403.19887", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "§1", "note": "Report: 'we release Jamba (12B active parameters, 52B total available parameters) under Apache 2.0 license'." }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract / §1", "note": "Report: 'We make the weights of our implementation of Jamba publicly available under a permissive license'; HF link ai21labs/Jamba-v0.1." } }, "scale": { "total_params": { "value": 52, "provenance": "stated", "ref": "Abstract / §1 / Table 1", "note": "Report: '12B active parameters, 52B total available parameters'; Table 1 lists 52B." }, "active_params_per_token": { "value": 12, "provenance": "stated", "ref": "Abstract / §1 / Table 1", "note": "Report: '12B active parameters'; Table 1 lists 12B." }, "layers": { "value": 32, "provenance": "derived", "note": "4 Jamba blocks × l=8 layers per block (§3.1) = 32 layers." }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Report does not disclose the model (embedding) dimension; only l, a:m, e, n, K are given (§3.1)." }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Report does not disclose FFN dimensions." }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "Cannot be derived: ffn_inner_dim and hidden_dim are both undisclosed." }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Report does not disclose query head count." }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "GQA is stated (§2) but the KV-head count / group size is not disclosed." }, "head_dim": { "value": null, "provenance": "unknown", "note": "Cannot be derived: hidden_dim and head counts are undisclosed." }, "vocab_size": { "value": 65536, "provenance": "derived", "note": "Report: 'The vocabulary size is 64K' (§2); 64K read as 64 × 1024 = 65536." }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Report is silent on input/output embedding tying; not part of the disclosed configuration." }, "variants": [] }, "block": { "block_type": { "value": "hybrid", "provenance": "derived", "note": "MoE applied every e=2 layers (§3.1): 16 of 32 layers carry routed-MoE FFNs, 16 carry dense MLPs — neither pattern reaches the 80% share required for dense or sparse-MoE (taxonomy Axis 1)." }, "moe": { "expert_count": { "value": 16, "provenance": "stated", "ref": "§3.1 (n=16) / §6.3", "note": "Report: 'n=16: total number of experts'." }, "experts_per_token": { "value": 2, "provenance": "stated", "ref": "§3.1 (K=2) / §6.3", "note": "Report: 'K=2: number of top experts used at each token'." }, "shared_experts": { "value": 0, "provenance": "inferred", "note": "Absence rule: §2 describes the MoE module completely as n experts per layer with a router choosing top-K per token; no shared/always-on expert is mentioned." }, "routing": { "value": "top-K routing", "provenance": "stated", "ref": "§2 / §3.1", "note": "Report: 'a router choosing the top K experts at each token'; the gating function (softmax/sigmoid) is not specified." }, "load_balancing": { "value": "load balancing (mechanism not specified)", "provenance": "stated", "ref": "§2", "note": "Report: 'load balancing for the MoE [14]' (Switch Transformers); aux-loss vs other mechanisms not specified." }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "Report does not discuss expert granularity (fine-grained/grouped)." } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2", "note": "Report: 'Other architecture details are standard, including grouped-query attention (GQA)'." }, "attention_layer_pattern": { "value": "4 blocks of 8 layers each (l=8); a:m = 1:7 — one GQA attention layer per block, seven Mamba layers; attention layers at positions 4, 12, 20, 28 (32 layers total, 4 attention layers)", "provenance": "stated", "ref": "§3.1 / Figure 1 / §5.2.1", "note": "Positions per Figure 8, which places the attention layers of the 24-layer 1.3B variant at layers 4, 12, 20; extended to the 32-layer flagship, and §5.2.1 confirms 'only 4 attention layers'." }, "position_encoding": { "method": { "value": "NoPE", "provenance": "stated", "ref": "§2", "note": "Report: 'positional embeddings or mechanisms like RoPE are not necessary, and so we do not use any explicit positional information'." }, "rope_base": { "value": null, "provenance": "n/a", "ref": "§2" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "§2" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "Report: trained on context lengths up to 1M tokens (§3.1); released model supports up to 256K (§3.1, §5.2) — deployed length lies within the trained length, so no extension technique is needed; none is disclosed." }, "factor": { "value": null, "provenance": "n/a", "ref": "§3.1" }, "target_length": { "value": null, "provenance": "n/a", "ref": "§3.1" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2 / §6.4", "note": "Report: 'we apply RMSNorm [53] in the Mamba layers' (§2) and 'added RMSNorm [53] to internal activations' (§6.4); attention-layer normalisation is not separately specified." }, "placement": { "value": null, "provenance": "unknown", "note": "Report is silent on pre/post placement of normalisation." }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: §2 enumerates the non-standard details (GQA, SwiGLU, load balancing, RMSNorm in Mamba, no positional info) and §6.4 exhaustively discusses stability; QK-normalisation is absent from both and never mentioned." } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§2", "note": "Report: 'SwiGLU activation function [45, 7, 50]'." }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Report designs no sink tokens; the §2 architecture description would surface a designed sink mechanism were it used. Any attention-sink behaviour is emergent." }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: logit softcapping never mentioned; §6.4's stability discussion covers RMSNorm on Mamba internals only." }, "other": { "value": [ "RMSNorm applied to internal activations of Mamba layers to prevent loss spikes when scaling to the 7B-based model (§6.4)" ], "provenance": "stated", "ref": "§6.4" } }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: report describes its block in detail and never mentions attention over prior layer outputs; standard sequential residual connections are the default mechanism." } }, "context": { "trained_length": { "value": 1000000, "provenance": "stated", "ref": "§3.1 / §5.2", "note": "Report: 'our Jamba implementation was successfully trained on context lengths of up to 1M tokens'." }, "deployed_length": { "value": 256000, "provenance": "stated", "ref": "§3.1 / §5.2", "note": "Report: 'The released model supports lengths of up to 256K tokens'." }, "extension_method": { "value": "none", "provenance": "inferred", "note": "Deployed 256K lies within the up-to-1M training length (§3.1); no extension method is disclosed (mirrors position_encoding.extension)." } }, "tokenizer": { "algorithm": { "value": "BPE", "provenance": "stated", "ref": "§2", "note": "Report: 'The tokenizer is trained with BPE [16, 44, 33]'." }, "notes": { "value": "Each digit is a separate token; the dummy space used in the Llama and Mistral tokenizers is removed for more consistent and reversible tokenization. Vocabulary size 64K (recorded in scale.vocab_size).", "provenance": "stated", "ref": "§2" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "Report discloses no token count for the released 52B model; ablation runs state 1.3B models at 250B tokens and 7B models at 50B tokens (§6.1, §6.3), which do not apply to the flagship." }, "data_composition": { "value": "In-house dataset containing text from the Web, books, and code, last updated March 2024; processing pipeline includes quality filters and deduplication; mixture proportions not disclosed.", "provenance": "stated", "ref": "§4" }, "curriculum": { "value": "none disclosed", "provenance": "inferred", "note": "Report describes no staged training, annealing, or cooldown; only ablation token counts are given (§6.1)." }, "optimizer": { "value": null, "provenance": "unknown", "note": "Report does not name the optimizer." }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "Report does not disclose the learning-rate schedule." }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "Report does not disclose batch-size scheduling." }, "precision": { "value": null, "provenance": "unknown", "note": "Report mentions int8 weights only for inference (§1, §3.1, §3.2); training precision is not disclosed." }, "parallelism": { "value": "In-house proprietary training framework with FSDP, tensor parallelism, sequence parallelism, and expert parallelism", "provenance": "stated", "ref": "§4" }, "hardware": { "value": "NVIDIA H100 GPUs", "provenance": "stated", "ref": "§4" }, "compute": { "value": null, "provenance": "unknown", "note": "Report does not disclose FLOPs or GPU-hours." } }, "post_training": { "sft": { "value": false, "provenance": "stated", "ref": "§1", "note": "Report: the released model 'did not go through alignment or instruction tuning'." }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "§1", "note": "Report explicitly states the released model did not go through alignment (§1)." }, "reasoning_training": { "value": "none disclosed", "provenance": "inferred", "note": "Base model without alignment or instruction tuning (§1); no reasoning-training method (CoT SFT, RLVR, test-time compute) is described. GSM8K is evaluated 3-shot CoT (§5.1), a prompting choice, not training." }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "Report describes pre-training on an in-house corpus (§4) with no teacher model or distillation mentioned." } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "All described training data (Web, books, code; §4) and evaluations (§5) are text; no non-text modality is described." }, "attachment": { "value": null, "provenance": "n/a", "ref": "§4" } }, "efficiency": { "kv_cache": { "value": "GQA attention on only 4 of 32 layers: report states an 8x smaller KV cache than a vanilla Transformer and 4GB at 256K context (16-bit) vs 32GB for Mixtral and 128GB for Llama-2 (Table 1)", "provenance": "stated", "ref": "§2 / Table 1" }, "quantization": { "value": "INT8 (inference): model fits in a single 80GB GPU with int8/8bit weights, including at contexts over 128K tokens", "provenance": "stated", "ref": "§1 / §3.1 / §3.2" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "Report does not mention speculative decoding, draft models, or multi-token prediction." }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "Report measures throughput 'without possible optimizations' (§3.2) and discloses no serving techniques." } }, "evaluation": { "benchmarks": [ { "name": { "value": "HellaSwag (10-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 87.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "WinoGrande (5-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 82.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ARC-E (0-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ARC-C (25-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 64.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "PIQA (zero-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 83.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "NQ (5-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 45.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "TruthfulQA (zero-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 46.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BoolQ (10-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "QuAC (zero-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 40.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GSM8K (3-shot CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 59.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HumanEval (pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 29.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU (5-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 67.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BBH (3-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 45.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "NQ (long-context, 3-shot F1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 0.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Avg (L-Eval long-context F1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 0.44, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "First production-grade hybrid Attention-SSM language model, interleaving Transformer and Mamba layers (prior hybrids were small-scale or lagged pure models).", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } }, { "text": { "value": "MoE applied to every other MLP layer (16 experts, top-2, load balancing), increasing model capacity without proportionally increasing active parameters or compute, and shown beneficial for the hybrid at 7B scale.", "provenance": "stated", "ref": "§1 / §3.1 / §6.3" }, "ref": { "value": "§1 / §3.1 / §6.3", "provenance": "stated", "ref": "§1 / §3.1 / §6.3" } }, { "text": { "value": "Hybrid reaches Transformer-level quality — comparable to Mixtral-8x7B and the larger Llama-2 70B on standard benchmarks — while achieving up to 3x the throughput and an 8x smaller KV cache.", "provenance": "stated", "ref": "Abstract / §1 / §5.1" }, "ref": { "value": "Abstract / §1 / §5.1", "provenance": "stated", "ref": "Abstract / §1 / §5.1" } }, { "text": { "value": "Long-context capability of 256K tokens — described as the longest context supported by a production-grade publicly available model at the time — with strong needle-in-a-haystack performance using only 4 attention layers.", "provenance": "stated", "ref": "§1 / §5.2" }, "ref": { "value": "§1 / §5.2", "provenance": "stated", "ref": "§1 / §5.2" } }, { "text": { "value": "Ablation-derived insights: explicit positional information is unnecessary in the hybrid (Mamba layers before attention provide implicit position), and RMSNorm on Mamba internal activations stabilises large-scale training.", "provenance": "stated", "ref": "§6.4 / §6.5" }, "ref": { "value": "§6.4 / §6.5", "provenance": "stated", "ref": "§6.4 / §6.5" } }, { "text": { "value": "Open release of weights under Apache 2.0, plus a plan to release checkpoints from ablation runs to encourage community study of the architecture.", "provenance": "stated", "ref": "Abstract / §1" }, "ref": { "value": "Abstract / §1", "provenance": "stated", "ref": "Abstract / §1" } } ], "assessment": { "value": "Every ingredient is pre-existing — the report itself cites H3, Hyena, StripedHyena, Block-State Transformer, and MoE-Mamba as earlier Attention-SSM hybrids — so Jamba's novelty is scale and integration: the first production-grade hybrid at 52B total / 12B active parameters, a 1:7 attention-to-Mamba recipe with MoE on every other layer, and open weights. The empirical findings (no positional encoding needed; RMSNorm on Mamba internals to prevent loss spikes) are genuine, reusable contributions, and the 256K-token context with a 4GB KV cache stood out against contemporaneous open models. Relative to atlas peers, Jamba is the anchor for hybrid sequence mixing (taxonomy Axis 2).", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "hybrid", "provenance": "derived", "note": "MoE applied every e=2 layers (§3.1): 16 of 32 layers have routed-MoE FFNs, 16 dense MLPs — neither reaches the 80% threshold of taxonomy Axis 1." }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "Per-layer pattern 1 attention : 7 Mamba (§3.1): 4 of 32 layers are attention — deliberate alternation below the 80% majority; taxonomy Axis 2 lists Jamba (1 attention : 7 Mamba) as the anchor for hybrid." }, "modality": { "value": "text-only", "provenance": "derived", "note": "Derived from modality.type (inferred text-only: all described data and evaluations are text, §4, §5)." }, "openness": { "value": "open-weights", "provenance": "derived", "note": "Weights released under Apache 2.0 (§1); in-house training data described but not released (§4)." }, "scale_class": { "value": "large", "provenance": "derived", "note": "Reference variant 52B total params: 10B ≤ p < 100B (taxonomy Axis 5)." }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "Framed as a general-purpose base model matching Mixtral-8x7B and Llama-2 70B across a broad sweep of standard academic benchmarks (Abstract, §1, §5.1); efficiency (single 80GB GPU) and 256K long context are emphasised secondary frames." } }, "prose": { "design_overview": { "value": "Jamba interleaves Transformer and Mamba layers into repeated 'Jamba blocks' — the report's term for l=8 layers mixed at a 1:7 attention-to-Mamba ratio — and adds mixture-of-experts to some MLPs. The released configuration stacks 4 blocks: 32 layers total, of which only 4 are GQA attention layers (positions 4, 12, 20, 28) and the rest are Mamba state-space layers; every other layer's MLP is replaced by an MoE with 16 experts and top-2 routing, giving 52B total but only 12B active parameters. The design deliberately trades attention for Mamba to shrink the KV cache (reported 8x smaller than a vanilla Transformer; 4GB at 256K context) and raise long-context throughput (3x Mixtral at 128K), while keeping enough attention for in-context learning — the report shows pure-Mamba models fail to follow few-shot formats, and the hybrid restores Transformer-like behaviour with as little as one attention layer in eight. Notable choices: no explicit positional encoding (Mamba layers are said to provide implicit position), RMSNorm inside Mamba layers for training stability, SwiGLU activations, a 64K-token BPE tokenizer with per-digit tokens, and GQA. Trained on contexts up to 1M tokens on H100s with FSDP/tensor/sequence/expert parallelism, the released base model serves 256K-token contexts from a single 80GB GPU with int8 weights and reaches parity with Mixtral-8x7B and Llama-2 70B on standard benchmarks. The report covers a single configuration and discloses few per-dimension numbers; most ablation evidence comes from 1.3B and 7B runs.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Mamba selective state-space layers (the SSM component of each block)", "provenance": "inferred", "note": "authored by atlas analyst; report cites Mamba [17] as the state-space model (§1, §2)" }, "origin": { "value": "Mamba (Gu & Dao, 2023)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "Sparsely-gated mixture-of-experts on MLP layers (n experts, top-K router, load balancing)", "provenance": "inferred", "note": "authored by atlas analyst; report cites sparsely-gated MoE [46] and Switch Transformers [14] (§1, §2)" }, "origin": { "value": "Switch Transformers / sparsely-gated MoE (Shazeer et al.)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "Transformer decoder architecture with self-attention layers", "provenance": "inferred", "note": "authored by atlas analyst; report cites the Transformer [51] (§1, §2)" }, "origin": { "value": "Transformer (Vaswani et al., 2017)", "provenance": "inferred", "note": "authored by atlas analyst" } } ], "influence_out": [ { "model_slug": { "value": "nvidia-nemotron-3-nano-30b-a3b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Hybrid Mamba-Transformer architecture (Mamba-2 + GQA interleaving)", "provenance": "inferred", "note": "mechanism as stated in nvidia-nemotron-3-nano-30b-a3b report's lineage_in" } } ], "notable_omissions": { "value": [ "Training token count for the released model (only the 1.3B/250B and 7B/50B ablation runs disclose tokens).", "Per-dimension configuration: hidden size, FFN size, query-head count, and KV-head/group size are all undisclosed (only l, a:m, e, n, K are given).", "Data mixture proportions (domains named as Web, books, code; no percentages).", "Optimizer, learning-rate schedule, batch schedule, and training precision.", "Total training compute (FLOPs or GPU-hours).", "MoE router gating function and load-balancing mechanism (Switch Transformers cited, no detail).", "Normalisation placement (pre/post) for attention layers.", "How the 256K deployed context relates to the up-to-1M training length (no extension technique described)." ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "ai21-jamba.json" }, { "slug": { "value": "allenai-olmo-2-7b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2501.00656", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/allenai-olmo-2-7b", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "OLMo 2", "provenance": "stated", "ref": "Title / Abstract" }, "variants": [ { "value": "7B", "provenance": "stated", "ref": "Abstract ('models at 7B and 13B scales'); metadata block lists OLMo-2-1124-7B first" }, { "value": "13B", "provenance": "stated", "ref": "Abstract ('models at 7B and 13B scales')" } ], "org": { "value": "OLMo Team, Allen Institute for AI (Ai2)", "provenance": "stated", "ref": "Title page byline ('OLMo Team'; '1] Allen Institute for AI')" }, "release_date": { "value": "2024-12-31", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date 2024-12-31 per export.arxiv.org API metadata (arXiv:2501.00656, published 2024-12-31T21:55:10Z)" }, "report_url": { "value": "https://arxiv.org/abs/2501.00656", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "Section 2 ('we release all training code, data, and recipes openly under the Apache 2.0 license wherever possible')", "note": "Stated for code/data/recipes 'wherever possible'; the model weights release is described as fully open without a separate license clause" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('We release all OLMo 2 artifacts openly—models at 7B and 13B scales, both pretrained and post-trained, including their full training data, training code and recipes')" } }, "scale": { "total_params": { "value": 7, "provenance": "stated", "ref": "Abstract ('models at 7B and 13B scales'); Section 2.2 ('deliver 7B and 13B parameter variants')", "note": "Report names the variants by their parameter scale and never prints an exact count (e.g. 6.9B); 7B is the reference variant (slug, metadata ordering, all deep-dive experiments run on 7B checkpoints)" }, "active_params_per_token": { "value": 7, "provenance": "derived", "note": "Dense model: active params per token = total params (7B)" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 4 (OLMo 2 7B: Layers 32)" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 4 (Hidden Size (d_model) 4096)" }, "ffn_inner_dim": { "value": 11008, "provenance": "stated", "ref": "Section 2.2 ('set the corresponding hidden size to approximately 8/3 d, but increased to the closest multiple of 128 (11,008 for our 7B model)')" }, "ffn_ratio": { "value": 2.7, "provenance": "derived", "note": "11008 / 4096 = 2.6875, rounded to one decimal" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 4 (Attention Heads 32)" }, "attention_heads_kv": { "value": 32, "provenance": "inferred", "note": "Report states 'Attention Heads 32' and never mentions GQA/MQA/KV-head grouping anywhere; MHA reading (absence rule) → KV heads = query heads = 32" }, "head_dim": { "value": 128, "provenance": "derived", "note": "4096 / 32 = 128" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "Report borrows the cl100k pretokenizer and vocabulary (Section 2.3.1) but never prints a vocabulary-size integer" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Input/output embedding tying never mentioned (only that embeddings are excluded from weight decay, Section 3.4.2)" }, "variants": [ { "name": { "value": "OLMo 2 13B", "provenance": "stated", "ref": "Abstract / Table 4" }, "total_params": { "value": 13, "provenance": "stated", "ref": "Abstract ('models at 7B and 13B scales'); Section 2.2 ('7B and 13B parameter variants')", "note": "Report names the variant by its parameter scale; no exact count printed" }, "layers": { "value": 40, "provenance": "stated", "ref": "Table 4 (OLMo 2 13B: Layers 40)" }, "hidden_dim": { "value": 5120, "provenance": "stated", "ref": "Table 4 (Hidden Size (d_model) 5120)" }, "ffn_inner_dim": { "value": 13696, "provenance": "derived", "note": "Stated rule (Section 2.2): SwiGLU hidden size ≈ 8/3 d, increased to the closest multiple of 128. 8/3 × 5120 = 13653.33 → closest multiple of 128 = 13696" }, "attention_heads_q": { "value": 40, "provenance": "stated", "ref": "Table 4 (Attention Heads 40)" }, "attention_heads_kv": { "value": 40, "provenance": "inferred", "note": "No KV-head grouping disclosed anywhere in the report; MHA reading → KV heads = query heads = 40" }, "context_length": { "value": 4096, "provenance": "stated", "ref": "Table 4 (Sequence Length 4096 for both variants)" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "stated", "ref": "Abstract ('OLMo 2 includes dense autoregressive models')" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "Abstract ('dense autoregressive models')" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "Abstract ('dense autoregressive models')" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "Abstract ('dense autoregressive models')" }, "routing": { "value": null, "provenance": "n/a", "ref": "Abstract ('dense autoregressive models')" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "Abstract ('dense autoregressive models')" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "Abstract ('dense autoregressive models')" } }, "attention_variant": { "value": "MHA", "provenance": "inferred", "note": "Report describes the attention block in detail (Table 3, Section 2.2) and never mentions GQA/MQA/MLA or KV-head sharing; standard multi-head attention with 32 query heads (absence rule)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "No per-layer attention pattern described; all layers use the same full attention (no sliding window, no hybrid)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: block described in detail (Section 2.2) with standard residual connections (h := x + RMSNorm(Attention(x))); no attention over prior layer outputs" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "Section 2.2 ('We replace absolute positional embeddings with rotary positional embeddings (RoPE)')" }, "rope_base": { "value": 500000, "provenance": "stated", "ref": "Table 3 (RoPE θ: 5·10^5); Section 2.2 ('We increase the RoPE θ to 500,000 from 10,000')" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "Report never states which dimensions receive RoPE" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "No context-extension method disclosed; training sequence length is 4096 (Table 4) and no extension beyond it is described" }, "factor": { "value": null, "provenance": "unknown" }, "target_length": { "value": null, "provenance": "unknown" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "Section 2.2 / 3.3.1 ('We use the RMSNorm variant of LayerNorm')" }, "placement": { "value": "post", "provenance": "stated", "ref": "Section 2.2 Eqs. 1-2 / 3.3.2 ('We normalize the outputs to the attention and feedforward (MLP) layers within each transformer block, instead of the inputs')", "note": "'Reordered norm': h := x + RMSNorm(Attention(x)); h_out := h + RMSNorm(MLP(h)) — RMSNorm applied to the sublayer output (post-sublayer), not the input" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "Section 2.2 / 3.3.2 ('we normalize the key and query projections with RMSNorm before calculating attention')" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "Section 2.2 / Table 3 (Activation: SwiGLU)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: architecture described in full detail (Section 2.2, Table 3); no designed sink tokens" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no attention-logit softcapping described" }, "other": { "value": [ "z-loss regularization (Table 3 gives weight 10^-5; Section 3.3.3 text writes the term as 10^-4 · log^2 Z — the report's two figures disagree)", "No bias terms anywhere in the architecture (Section 2.2)", "Trainer-side loss masking of input sequences containing 32+ repeated n-grams, in addition to curation-time filtering (Section 3.1)", "Truncated normal initialization, mean 0, std 0.02 (Sections 2.3 / 3.2)" ], "provenance": "stated", "ref": "Sections 2.2 / 3.1 / 3.2 / 3.3.3 / Table 3" } } }, "context": { "trained_length": { "value": 4096, "provenance": "stated", "ref": "Table 4 (Sequence Length 4096 for both variants)" }, "deployed_length": { "value": null, "provenance": "unknown", "note": "Report never states the served inference context length; no context extension is described" }, "extension_method": { "value": null, "provenance": "unknown", "note": "No extension disclosed; training ran at 4096 and the report is silent on any longer deployment" } }, "tokenizer": { "algorithm": { "value": "byte-level BPE (cl100k / tiktoken-style)", "provenance": "stated", "ref": "Section 2.3.1 ('We borrow pre-tokenizer and vocabulary from cl100k, the tokenizer developed for GPT-3.5 and GPT-4')" }, "notes": { "value": "cl100k vocabulary plus the PII-masking special tokens (|||PHONE_NUMBER|||, |||EMAIL_ADDRESS|||, |||IP_ADDRESS|||) kept from earlier OLMo tokenizers for backwards compatibility with early Dolma sources; base models use the GPT2Tokenizer class with custom pre-tokenization logic; a tokenizer mismatch between the initial Instruct Preview and the base models led to retraining the Instruct models (Appendix B.3)", "provenance": "stated", "ref": "Section 2.3.1 / Appendix B.3" } }, "training": { "tokens": { "value": 4.05, "provenance": "stated", "ref": "Section 2.1 ('In total, OLMo 2 7B is trained on 4.05 trillion tokens (3.90 trillion for pretraining stage)')", "note": "3.90T pretraining + 3 × 50B mid-training (souped) = 4.05T" }, "data_composition": { "value": "Two-stage mix. Pretraining: OLMo 2 Mix 1124, ~3.9T tokens, >95% web (DCLM-Baseline web 3.71T, StarCoder filtered code 83.0B, peS2o papers 58.6B, arXiv 20.8B, OpenWebMath 12.2B, Algebraic Stack 11.8B, Wikipedia/Wikibooks 3.7B). Mid-training: Dolmino Mix 1124 (50B for 7B), high-quality web (DCLM FastText top-7% + FineWeb≥2, ~47% of mix) plus FLAN, StackExchange Q&A, peS2o, Wikipedia, and a math mix (TuluMath synthetic 230M, DolminoSynthMath 28.7M, TinyGSM-MIND 6.48B, MathCoder2-synthetic 3.87B, Metamath, CodeSearchNet, GSM8K train). Exact 50B/100B/300B mix percentages in Table 13.", "provenance": "stated", "ref": "Section 2.1 / Tables 1, 2, 13" }, "curriculum": { "value": "Two-stage: pretraining on web data (cosine LR decay calibrated over 5T tokens, truncated at 4T for the 7B), then mid-training ('annealing') on Dolmino Mix 1124 with linear LR decay to zero, run multiple times with different data-order seeds and averaged (checkpoint souping: 3×50B runs for 7B; 3×100B + 1×300B for 13B)", "provenance": "stated", "ref": "Sections 2.3 / 3.5 / 4.3" }, "optimizer": { "value": "AdamW with epsilon lowered from 1e-5 to 1e-8; weight decay 0.1 applied multiplicatively (1 − 0.1·lr per step), excluded for embeddings; gradient clipping 1.0", "provenance": "stated", "ref": "Sections 3.4.1 / 3.4.2 / Table 4" }, "lr_schedule": { "value": "Linear warmup from 0 over 2000 steps to peak 3.0e-4 (7B; 13B peak 9.0e-4), then cosine decay calibrated to reach 10% of peak after 5T tokens (7B truncated at 4T before mid-training); mid-training stage linearly decays LR to zero", "provenance": "stated", "ref": "Table 4 / Sections 2.3 / 3.5" }, "batch_schedule": { "value": "Batch size 1024 (7B) / 2048 (13B) sequences at sequence length 4096; mid-training uses the same batch size (Table 13 note: 13B runs 100B-token anneals to keep the same number of steps)", "provenance": "stated", "ref": "Table 4 / Section 4.3" }, "precision": { "value": null, "provenance": "unknown", "note": "Master training precision (BF16/FP8) never stated; report only mentions low-precision training in passing (Section 3.2) and precision differences in z-loss implementations (Section 3.3.3)" }, "parallelism": { "value": null, "provenance": "unknown", "note": "No parallelism configuration disclosed (no TP/PP/FSDP statements); Section 6 discusses Beaker scheduling, torch.compile, NCCL/GLOO backends and throughput optimizations but no sharding scheme" }, "hardware": { "value": "Two Ai2 clusters: Jupiter (Cirrascale, Austin TX) — 1,024 NVIDIA H100 80GB HBM3 GPUs across 128 nodes, InfiniBand 400 Gbps/card, RDMA; Augusta (Google Cloud, Council Bluffs IA) — 160 A3 Mega VMs each with 8 NVIDIA H100 (1,280 GPUs), GPUDirect-TCPXO. Bulk of 7B training on Jupiter, bulk of 13B on Augusta.", "provenance": "stated", "ref": "Section 6.1" }, "compute": { "value": "OLMo 2 7B ≈ 1.8 × 10^23 FLOPs; OLMo 2 13B ≈ 4.6 × 10^23 FLOPs (Table 6, Kaplan 6×tokens×params approximation); pretraining energy ≈ 391 MWh, ≈ 154 tCO2eq, ≈ 1.1M liters water for both models (Table 19)", "provenance": "stated", "ref": "Table 6 / Section 6.5 / Table 19" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "Section 5 (Tülu 3 recipe, SFT stage; final SFT mix has 939,104 prompts, Section 5.2)" }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "Section 5 ('preference tuning with Direct Preference Optimization (DPO; Rafailov et al., 2024) and on-policy preference data')", "note": "Preference finetuning via DPO on synthetic on-policy preference data (366.7k prompts 7B / 377.7k 13B, GPT-4o-2024-08-06 judge, 20-model response pool); the final RLVR stage (PPO against verifiable rewards) is recorded under reasoning_training" }, "reasoning_training": { "value": "RLVR: Reinforcement Learning with Verifiable Rewards — PPO (Schulman et al. 2017) against rule-verifiable rewards (GSM8K and MATH training sets plus constraint prompts from Lambert et al. 2024); value function initialized from reward models trained on the on-policy preference data; multi-stage RLVR for the 13B (GSM8K pass, then MATH pass); RLVR hyperparameters in Table 18 (β sweep 0.03–0.1, 100k–200k episodes, max 2,048 response tokens)", "provenance": "stated", "ref": "Section 5 / Table 18" }, "distillation": { "value": "none", "provenance": "inferred", "note": "No teacher-model distillation described. Synthetic SFT (PersonaHub/GPT-4o math data) and preference data (GPT-4o judge over a 20-model pool, Qwen2.5-7B rewrites) are data generation, not distillation (per schema Section 8)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes only text input/output; all evaluation tasks (OLMES, GSM8K, MMLU, etc.) are text-based; no vision/audio modality described" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "Section 2.4", "note": "text-only model; no non-text modality attaches" } }, "efficiency": { "kv_cache": { "value": "MHA: KV cache proportional to 32 KV heads (all query heads) per token; no KV sharing. Report does not discuss the KV cache or its size.", "provenance": "derived", "note": "Derived from attention_variant (MHA, 32 heads); report itself is silent on KV-cache design" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide", "note": "No inference quantisation formats disclosed" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "Abstract / metadata block", "note": "Only the Ai2 Playground research demo is mentioned; no serving optimisations disclosed" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (OLMES CF)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 63.7, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "ARC Challenge", "provenance": "stated", "ref": "Table 6 (column 'ARC C') / Section 4 text ('Arc challenge: 72.6→79.8')" }, "value": { "value": 79.8, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "HellaSwag", "provenance": "stated", "ref": "Table 6 (column 'HSwag')" }, "value": { "value": 83.8, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Winogrande", "provenance": "stated", "ref": "Table 6 (column 'WinoG')" }, "value": { "value": 77.2, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Natural Questions (OLMES-Gen, F1)", "provenance": "stated", "ref": "Table 6 (column 'NQ')" }, "value": { "value": 36.9, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "DROP (OLMES-Gen, F1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 60.8, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "AGIEval", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 50.4, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GSM8K", "provenance": "stated", "ref": "Table 6 (held-out column)" }, "value": { "value": 67.5, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMLU Pro", "provenance": "stated", "ref": "Table 6 (held-out column 'MMLU PRO')" }, "value": { "value": 31.0, "provenance": "stated", "ref": "Table 6 (OLMo 2 7B row)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GSM8K (OLMo 2 7B Instruct)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 85.1, "provenance": "stated", "ref": "Table 7 (OLMo 2 7B Instruct row)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "IFEval (OLMo 2 7B Instruct)", "provenance": "stated", "ref": "Table 7 (column 'IFE')" }, "value": { "value": 72.3, "provenance": "stated", "ref": "Table 7 (OLMo 2 7B Instruct row)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "MATH (OLMo 2 7B Instruct)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 32.5, "provenance": "stated", "ref": "Table 7 (OLMo 2 7B Instruct row)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } } ] }, "contributions": { "claimed": [ { "text": { "value": "A fully open family of 7B and 13B models (base + Instruct) with all training data, code, recipes, logs and thousands of intermediate checkpoints released; base models sit on the performance-to-compute Pareto frontier, often matching Llama 3.1 and Qwen 2.5 with fewer FLOPs", "provenance": "stated", "ref": "Abstract / Section 1" }, "ref": { "value": "Abstract / Section 1", "provenance": "stated", "ref": "Abstract / Section 1" } }, { "text": { "value": "A training-stability package: repeated n-gram filtering and loss masking, truncated-normal initialization (mean 0, std 0.02), RMSNorm with reordered (output) placement, QK-norm, z-loss, excluding embeddings from weight decay, and AdamW epsilon 1e-8", "provenance": "stated", "ref": "Section 3" }, "ref": { "value": "Section 3", "provenance": "stated", "ref": "Section 3" } }, { "text": { "value": "A mid-training recipe on the Dolmino Mix 1124 with micro-annealing as a low-cost technique to evaluate individual data sources, plus checkpoint souping over multiple anneal runs with different data orders", "provenance": "stated", "ref": "Section 4" }, "ref": { "value": "Section 4", "provenance": "stated", "ref": "Section 4" } }, { "text": { "value": "OLMo 2-Instruct built on the Tülu 3 recipe (SFT, DPO, RLVR) with permissively-licensed data and a multi-stage RLVR protocol that consistently improves both reward and downstream evaluations at both scales", "provenance": "stated", "ref": "Section 5" }, "ref": { "value": "Section 5", "provenance": "stated", "ref": "Section 5" } }, { "text": { "value": "Infrastructure as a research catalyst: Beaker workload portability, torch.compile-based throughput, asynchronous bookkeeping, explicit garbage collection, and GPU health checks that improved cluster utilization and reduced failure rates", "provenance": "stated", "ref": "Section 6" }, "ref": { "value": "Section 6", "provenance": "stated", "ref": "Section 6" } } ], "assessment": { "value": "The block is a conservative evolution of OLMo 1 / OLMo-0424: the only structural changes are QK-norm, reordered output-side RMSNorm, z-loss, RoPE base 500K, and no embedding weight decay — all stability techniques borrowed from PaLM/Chameleon/Dehghani et al. lineage, with no new mixing or attention mechanism. Against atlas peers, the architecture is closest to Llama 3.1 (dense, RoPE, RMSNorm, SwiGLU) but keeps plain MHA at 4K context (no GQA, no 128K extension), and to Phi-3 in scale class; its genuinely distinctive contribution is the fully-open data/recipe release (open-weights-open-data, the first in the atlas at this scale) plus the micro-annealing data-curation methodology. The 2-point average gains over OLMo-0424 are attributed by the report to the mid-training mix and souping rather than to the architecture changes.", "provenance": "inferred", "note": "comparison against atlas entries (meta-llama-3.1, microsoft-phi-3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "Report states 'dense autoregressive models' (Abstract); no routing anywhere (block_type dense)" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "Uniform MHA in all 32 layers; no sliding-window, sparse, or SSM pattern disclosed (attention_layer_pattern uniform)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type: report describes no non-text modality" }, "openness": { "value": "open-weights-open-data", "provenance": "derived", "note": "Abstract: models released openly 'including their full training data, training code and recipes'; data mixes (OLMo 2 Mix 1124, Dolmino Mix 1124) are released artifacts (taxonomy.md Axis 4)" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "7B total params (reference variant): 1B ≤ p < 10B (taxonomy.md Axis 5)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "Framed as a general-purpose fully-open family competitive with Llama 3.1 / Qwen 2.5 / Gemma 2 across broad benchmark suites (Abstract, Section 1); math-targeted mid-training and RLVR are capability ingredients, not the lead framing" } }, "prose": { "design_overview": { "value": "OLMo 2 is a dense decoder-only transformer family (7B/32 layers, 13B/40 layers) that deliberately keeps the OLMo 1 block structure and changes only what training-stability experiments demanded. Each layer is x + RMSNorm(Attention(x)) then x + RMSNorm(MLP(x)): RMSNorm sits on the sublayer outputs ('reordered norm', from Liu et al. 2021), queries and keys get their own RMSNorm QK-norm before attention, z-loss (10^-5) tames logit growth, and the RoPE base is raised to 500,000. There is no KV-head sharing — plain MHA with head count equal to layer width/128 — and the context window stays at 4,096 tokens with no extension. Training is the more distinctive half: two stages, ~3.9T web-heavy tokens (OLMo 2 Mix 1124) then a 50B-token anneal on Dolmino Mix 1124 (quality-filtered web plus synthetic math like TinyGSM-MIND and TuluMath), repeated with three different data orders and averaged into a checkpoint soup; AdamW runs with epsilon 1e-8, no weight decay on embeddings, and gradient clipping 1.0. The 13B doubles the batch and anneals on 100B/300B mixes. Post-training follows Tülu 3: SFT on 939k prompts, DPO on GPT-4o-judged on-policy preference data, then multi-stage RLVR (PPO against verifiable GSM8K/MATH rewards, value function seeded from reward models). The significance is not the block — it is that every artifact, data point, and log is released, making it the first fully-open record in the atlas at this scale.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "RoPE base frequency 500,000 (raised from 10,000)", "provenance": "stated", "ref": "Section 2.2 ('matching Grattafiori et al. (2024)')" }, "origin": { "value": "Llama 3 (Grattafiori et al. 2024)", "provenance": "stated", "ref": "Section 2.2" } }, { "mechanism": { "value": "QK-norm (RMSNorm on query and key projections before attention)", "provenance": "stated", "ref": "Section 2.2 ('Following Dehghani et al. (2023b)')" }, "origin": { "value": "Dehghani et al. 2023 (Scaling ViT to 22B)", "provenance": "stated", "ref": "Section 2.2" } }, { "mechanism": { "value": "Reordered layer norm (norm on sublayer outputs instead of inputs)", "provenance": "stated", "ref": "Section 3.3.2 ('Liu et al. (2021) first introduced the idea of reordering layer norm')" }, "origin": { "value": "Liu et al. 2021 (not in atlas)", "provenance": "stated", "ref": "Section 3.3.2" } }, { "mechanism": { "value": "z-loss regularization", "provenance": "stated", "ref": "Section 3.3.3 ('Following Chowdhery et al. (2022), Chameleon Team (2024), and Wortsman et al. (2023)')" }, "origin": { "value": "PaLM (Chowdhery et al. 2022, not in atlas)", "provenance": "stated", "ref": "Section 3.3.3" } }, { "mechanism": { "value": "cl100k tokenizer (pretokenizer and vocabulary)", "provenance": "stated", "ref": "Section 2.3.1 ('the tokenizer developed for GPT-3.5 and GPT-4')" }, "origin": { "value": "OpenAI GPT-3.5 / GPT-4 (not in atlas)", "provenance": "stated", "ref": "Section 2.3.1" } }, { "mechanism": { "value": "Post-training recipe: SFT, on-policy DPO, RLVR with value-initialized PPO", "provenance": "stated", "ref": "Sections 2.5 / 5 ('we follow the Tülu 3 recipe (Lambert et al., 2024)')" }, "origin": { "value": "Tülu 3 (Lambert et al. 2024, not in atlas)", "provenance": "stated", "ref": "Section 5" } } ], "influence_out": [ { "model_slug": { "value": "allenai-olmo-3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "RMSNorm on outputs, QK-norm, RoPE 5e5, z-loss stability package", "provenance": "inferred", "note": "mechanism as stated in allenai-olmo-3 report's lineage_in" } } ], "notable_omissions": { "value": [ "No exact parameter counts (variants named only as '7B' and '13B')", "No vocabulary-size integer (cl100k borrowed; never quantified)", "Training precision (BF16/FP8) not stated; parallelism configuration not disclosed", "No context extension; deployed inference context length not stated", "Input/output embedding tying not disclosed; partial-RoPE dimension split not disclosed", "z-loss weight discrepancy inside the report: Table 3 prints 10^-5, Section 3.3.3 text writes 10^-4 · log^2 Z", "13B FFN inner dim not printed (derived from the stated 8/3d-to-multiple-of-128 rule)", "No KV-cache sizing, inference quantisation, speculative decoding, or serving details", "Extraction note: ar5iv HTML conversion garbles LaTeX math and table structures; headline numbers verified against the text (Tables 4, 6, 7, 33 survive intact)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "allenai-olmo-2-7b.json" }, { "slug": { "value": "allenai-olmo-3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2512.13961", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/allenai-olmo-3", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "OLMo 3", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "7B", "provenance": "stated", "ref": "Abstract / Table 33" }, { "value": "32B", "provenance": "stated", "ref": "Abstract / Table 33" } ], "org": { "value": "Allen Institute for AI (Olmo Team)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2025-12-15", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date per export.arxiv.org API (arXiv:2512.13961; v2 dated 2026-04-14)" }, "report_url": { "value": "https://arxiv.org/abs/2512.13961", "provenance": "stated", "ref": "pipeline" }, "license": { "provenance": "unknown", "note": "Report does not name a license for the weights; data licensing discussed" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract / §1 (whole model flow released: checkpoints, code, Dolma 3 data mixes)" } }, "scale": { "total_params": { "value": 32, "provenance": "stated", "ref": "Abstract (flagship 32B); exact parameter counts not disclosed" }, "active_params_per_token": { "value": 32, "provenance": "derived", "note": "Dense model: active = total (32B reference variant)" }, "layers": { "value": 64, "provenance": "stated", "ref": "Table 33" }, "hidden_dim": { "value": 5120, "provenance": "stated", "ref": "Table 33" }, "ffn_inner_dim": { "provenance": "unknown" }, "ffn_ratio": { "provenance": "unknown" }, "attention_heads_q": { "value": 40, "provenance": "stated", "ref": "Table 33" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 33 (GQA at 32B; 7B uses 32/32 MHA)" }, "head_dim": { "value": 128, "provenance": "derived", "note": "5120 / 40 = 128" }, "vocab_size": { "provenance": "unknown", "note": "Tokenizer is the OLMo 2 cl100k-derived BPE; size not stated" }, "embedding_tied": { "provenance": "unknown" }, "variants": [ { "name": { "value": "7B", "provenance": "stated", "ref": "Abstract / Table 33" }, "total_params": { "value": 7, "provenance": "stated", "ref": "name convention" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 33" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 33" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 33" }, "attention_heads_kv": { "value": 32, "provenance": "stated", "ref": "Table 33 (MHA at 7B)" }, "context_length": { "value": 65536, "provenance": "inferred", "note": "Long-context stage trains at 65,536 (§3.2 / Table 35); pretrain/midtrain at 8,192" } }, { "name": { "value": "32B", "provenance": "stated", "ref": "Abstract / Table 33" }, "total_params": { "value": 32, "provenance": "stated", "ref": "name convention" }, "layers": { "value": 64, "provenance": "stated", "ref": "Table 33" }, "hidden_dim": { "value": 5120, "provenance": "stated", "ref": "Table 33" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "value": 40, "provenance": "stated", "ref": "Table 33" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 33 (GQA at 32B)" }, "context_length": { "value": 65536, "provenance": "inferred", "note": "Long-context stage trains at 65,536 (§3.2 / Table 35); pretrain/midtrain at 8,192" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "All layers dense (no routing disclosed, Table 33)" }, "moe": { "expert_count": { "provenance": "n/a", "ref": "Table 33 dense" }, "experts_per_token": { "provenance": "n/a", "ref": "Table 33 dense" }, "shared_experts": { "provenance": "n/a", "ref": "Table 33 dense" }, "routing": { "provenance": "n/a", "ref": "Table 33 dense" }, "load_balancing": { "provenance": "n/a", "ref": "Table 33 dense" }, "expert_granularity": { "provenance": "n/a", "ref": "Table 33 dense" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§3.2", "note": "Sliding-window attention on 3/4 of layers (window 4,096) with full attention on the last layer and the remaining quarter" }, "attention_layer_pattern": { "value": "Sliding window on 3 of every 4 layers (window 4,096), full attention on the 4th; last layer full attention", "provenance": "stated", "ref": "§3.2" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "Table 33 / §3.2" }, "rope_base": { "value": 500000, "provenance": "stated", "ref": "Table 33" }, "partial_rope": { "provenance": "unknown" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "note": "YaRN applied to full-attention layers only (§3.2)", "ref": "§3.2 / Table 35" }, "factor": { "provenance": "unknown" }, "target_length": { "value": 65536, "provenance": "stated", "ref": "§3.2 / Table 35" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "Table 33 / §3.2 (on outputs)" }, "placement": { "value": "post", "provenance": "stated", "note": "RMSNorm applied to sub-layer outputs (Eqs. in §3.2)", "ref": "§3.2" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "Table 33 / §3.2 (QK-Norm)" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "Table 33 / §3.2" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: detailed architecture description, no sink tokens" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule" }, "other": { "value": [ "z-loss 1e-5", "no weight decay on embeddings" ], "provenance": "stated", "ref": "Table 33 / §3.2" } } }, "context": { "trained_length": { "value": 8192, "provenance": "stated", "ref": "Table 35 (pretrain/midtrain sequence length)" }, "deployed_length": { "value": 65536, "provenance": "stated", "ref": "§3.2 / Table 35 (long-context stage trains directly at 65,536)" }, "extension_method": { "value": "YaRN applied to full-attention layers only; best-fit document packing; intra-document masking; 8-way context parallelism", "provenance": "stated", "ref": "§3.2" } }, "tokenizer": { "algorithm": { "value": "BPE (cl100k-derived)", "provenance": "stated", "ref": "§2 (same tokenizer as OLMo 2)" }, "notes": { "value": "Vocabulary size not stated", "provenance": "stated", "ref": "pipeline" } }, "training": { "tokens": { "value": 5.5, "provenance": "stated", "ref": "Table 35 (32B pretraining; 7B uses 5.93T)" }, "data_composition": { "value": "Dolma 3 pretraining mix; midtrain stages (2×100B soup candidates); long-context stage 100B (32B); Dolci Think/Instruct/RL-Zero data suites for post-training (Table 35 / §3–6)", "provenance": "stated", "ref": "Table 35 / §3" }, "curriculum": { "value": "Pretrain → midtrain (2×100B, checkpoint-souped) → long-context stage (100B at 65K)", "provenance": "stated", "ref": "§3.2 / Table 35" }, "optimizer": { "provenance": "unknown", "note": "Adam-family implied by recipe but name not stated" }, "lr_schedule": { "value": "Cosine truncated at 5.5T tokens; peak LR 6.0e-4 (32B)", "provenance": "stated", "ref": "Table 35" }, "batch_schedule": { "value": "1,024 instances per batch (8,388,608 tokens)", "provenance": "stated", "ref": "Table 35" }, "precision": { "value": "BF16", "provenance": "stated", "ref": "§3.2" }, "parallelism": { "value": "HSDP + DP-replication + Llama 3-style context parallelism", "provenance": "stated", "ref": "§3.2" }, "hardware": { "value": "H100 80GB × 1,024 (pretraining)", "provenance": "stated", "ref": "Table 35 / §2.4" }, "compute": { "value": "~56 days end-to-end; $2.75M at $2/H100-hr; 1,960 TPS/GPU (32B)", "provenance": "stated", "ref": "§2.4 / Table 35" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.2 (Dolci Think SFT); §5 (Instruct SFT)" }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "§4.3 / §5 (Think: DPO then OlmoRL; Instruct: length-normalized DPO)" }, "reasoning_training": { "value": "Think: SFT → DPO → OlmoRL (GRPO + DAPO/Dr.GRPO-style RLVR: zero-gradient filtering, active sampling, token-level loss, no KL, clip-higher, truncated importance sampling, no std-dev normalization); RL-Zero: RLVR directly from Base across 4–5 domains; Instruct: SFT + function calling + length-normalized DPO + RLVR", "provenance": "stated", "ref": "§4–6" }, "distillation": { "value": "none", "provenance": "inferred", "note": "SFT data generated with Qwen3-32B chosen responses is teacher-generated data, not distillation (schema note); report frames training as from-scratch with open data" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes text-only evaluation and no non-text modality" }, "attachment": { "provenance": "n/a", "ref": "§2 model description" } }, "efficiency": { "kv_cache": { "value": "GQA at 32B (8 KV heads) plus sliding-window layers (window 4,096) shrink KV cache vs full attention; YaRN only on full-attention layers", "provenance": "derived", "note": "from attention pattern (Table 33 / §3.2)" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MATH", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 96.2, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "AIME 2024", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 80.6, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "AIME 2025", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 78.1, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "OMEGA", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 53.4, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "BigBenchHard", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 88.6, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "ZebraLogic", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 80.1, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "AGI Eval English", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 89.2, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "HumanEvalPlus", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 91.5, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "MBPP+", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 68.3, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "LiveCodeBench v3", "provenance": "stated", "ref": "Table 14" }, "value": { "value": 83.3, "provenance": "stated", "ref": "Table 14" }, "ref": { "value": "Table 14 (OLMo 3.1 32B Think column)", "provenance": "stated", "ref": "Table 14" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Fully-open model flow (checkpoints, code, Dolma 3 data, recipes) at 7B and 32B", "provenance": "stated", "ref": "Abstract / §1" }, "ref": { "value": "Abstract / §1", "provenance": "stated", "ref": "Abstract / §1" } }, { "text": { "value": "OlmoRL: an open RLVR algorithm (GRPO + DAPO/Dr.GRPO-style) with documented design choices", "provenance": "stated", "ref": "§6 / Abstract" }, "ref": { "value": "§6 / Abstract", "provenance": "stated", "ref": "§6 / Abstract" } }, { "text": { "value": "Long-context extension to 65K via YaRN applied to full-attention layers only, with best-fit packing and intra-document masking", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } }, { "text": { "value": "RL-Zero setup decontaminated from pretraining/midtraining data with spurious-reward negative controls", "provenance": "stated", "ref": "§6" }, "ref": { "value": "§6", "provenance": "stated", "ref": "§6" } }, { "text": { "value": "Sliding-window (3:1) + full-attention hybrid pattern at 32B", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } } ], "assessment": { "value": "Directly extends the OLMo 2 record's design package (RMSNorm on outputs, QK-norm, RoPE 5e5, z-loss) with GQA at 32B and a 3:1 sliding-window/full-attention hybrid — the first SWA hybrid in the dense line of the atlas. The genuinely notable novelty is the openness of the RLVR pipeline (OlmoRL) and the RL-Zero decontamination methodology, which the atlas records under post-training and contributions rather than block design.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "from block: all layers dense" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "3/4 of layers (75%) sliding-window, 1/4 full attention — below the 80% threshold; deliberate 3:1 alternation (§3.2)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights-open-data", "provenance": "derived", "note": "open_weights stated true AND data (Dolma 3, Dolci) released (§1)" }, "scale_class": { "value": "large", "provenance": "derived", "note": "32B reference variant" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "leads with reasoning/coding/long-context (Think flagship) but framed as a general model flow (Base/Think/Instruct/RL-Zero)" } }, "prose": { "design_overview": { "value": "OLMo 3 is AI2's fully-open dense model flow at 7B and 32B. The 32B flagship keeps the OLMo 2 stability package — RMSNorm on sub-layer outputs, QK-norm, RoPE base 500,000, z-loss, no embedding weight decay — and adds two structural changes: GQA with 8 KV heads at 32B (7B stays MHA), and a 3:1 hybrid attention pattern where three of every four layers use sliding-window attention (window 4,096) and the fourth plus the final layer use full attention. Context grows in three stages — 8K pretrain, 8K midtrain with checkpoint souping, then a 65K long-context stage using YaRN on full-attention layers only, best-fit document packing, intra-document masking, and 8-way context parallelism. Post-training is equally staged: Dolci Think SFT, DPO, then OlmoRL — a fully open GRPO-family RLVR algorithm. The report's distinctive claim is completeness of disclosure: data, checkpoints, and recipes all released.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "RMSNorm on outputs, QK-norm, RoPE 5e5, z-loss stability package", "provenance": "stated", "ref": "§3.2 / Table 33 (continuity with OLMo 2)" }, "origin": { "value": "OLMo 2", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "RoPE base 500,000", "provenance": "stated", "ref": "Table 33" }, "origin": { "value": "Llama 3 line", "provenance": "inferred", "note": "report states the value; lineage naming is analyst judgement" } }, { "mechanism": { "value": "GRPO-family RLVR with active sampling, token-level loss, clip-higher (OlmoRL)", "provenance": "stated", "ref": "§6 (cites GRPO/DAPO/Dr.GRPO)" }, "origin": { "value": "GRPO / DAPO / Dr.GRPO line", "provenance": "stated", "ref": "§6" } } ], "influence_out": [], "notable_omissions": { "value": [ "No exact parameter counts beyond 7B/32B", "No vocabulary size stated (cl100k-derived tokenizer)", "No pretraining optimizer name", "No head_dim stated (derived here)", "No FP8/quantization/serving/KV-cache discussion", "License for weights not named", "Source stored as PDF extraction (ar5iv fatal error banner; tables partially disassembled by pymupdf but values verified)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "allenai-olmo-3.json" }, { "slug": { "value": "arcee-ai-trinity-large-400b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2602.17004", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/arcee-ai-trinity-large-400b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Trinity", "provenance": "stated", "ref": "Abstract, §1 ('the Trinity family of open-weight Mixture-of-Experts (MoE) language models')" }, "variants": [ { "value": "Trinity Nano", "provenance": "stated", "ref": "Abstract (6B total / 1B activated)" }, { "value": "Trinity Mini", "provenance": "stated", "ref": "Abstract (26B total / 3B activated)" }, { "value": "Trinity Large", "provenance": "stated", "ref": "Abstract (400B total / 13B activated)" } ], "org": { "value": "Arcee AI", "provenance": "stated", "ref": "Title, byline (with Prime Intellect and DatologyAI collaborators)" }, "release_date": { "value": "2026-02-19", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date 2026-02-19 per export.arxiv.org API metadata (arXiv:2602.17004)" }, "report_url": { "value": "https://arxiv.org/abs/2602.17004", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "license never named (checkpoints linked at huggingface.co/arcee-ai)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('The model checkpoints are available at https://huggingface.co/arcee-ai'); §1 ('open-weight foundations')" } }, "scale": { "total_params": { "value": 400, "provenance": "stated", "ref": "Abstract, §1, Table 2" }, "active_params_per_token": { "value": 13, "provenance": "stated", "ref": "Abstract, §1, Table 2" }, "layers": { "value": 60, "provenance": "stated", "ref": "Table 2 (Transformer layers 60; 6 initial dense layers)" }, "hidden_dim": { "value": 3072, "provenance": "stated", "ref": "Table 2 (Model dim 3072), §3.4.1" }, "ffn_inner_dim": { "value": 3072, "provenance": "stated", "ref": "Table 2 (Expert size 3072), §3.4.1", "note": "per-expert FFN intermediate dim; the 6 initial dense layers use FFN intermediate dim 12288 (Table 2)" }, "ffn_ratio": { "value": 1.0, "provenance": "derived", "note": "per-expert ratio: 3072 / 3072 = 1.0 (dense-layer FFN ratio is 12288 / 3072 = 4.0)" }, "attention_heads_q": { "value": 48, "provenance": "stated", "ref": "Table 2 (Attention heads (hq) 48), §3.4.1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 2 (KV heads (hkv) 8), §3.4.1" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 2 (Per-head dim (dh) 128), §3.4.1", "note": "stated directly; note hidden/heads = 3072/48 = 64, which does not match the stated per-head dim" }, "vocab_size": { "value": 200000, "provenance": "stated", "ref": "§2.1.2 ('We selected 200,000 tokens')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "input/output embedding tying not disclosed (AdamW used for 'the embedding and output layers', §3.4.2, but tying not stated)" }, "variants": [ { "name": { "value": "Trinity Nano", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 6, "provenance": "stated", "ref": "Abstract" }, "layers": { "value": 56, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 1024, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "value": 3072, "provenance": "stated", "ref": "Table 2 (Expert size 256; FFN intermediate dim 3072 for dense layers)" }, "attention_heads_q": { "value": 8, "provenance": "stated", "ref": "Table 2" }, "attention_heads_kv": { "value": 2, "provenance": "stated", "ref": "Table 2" }, "context_length": { "value": 256000, "provenance": "stated", "ref": "§3.4.2 ('We train to 256k context for inference at 128k')", "note": "trained at 256K, served at 128K" } }, { "name": { "value": "Trinity Mini", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 26, "provenance": "stated", "ref": "Abstract" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 2048, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "value": 6144, "provenance": "stated", "ref": "Table 2 (FFN intermediate dim 6144; expert size 1024)" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 2" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "Table 2" }, "context_length": { "value": 128000, "provenance": "stated", "ref": "§3.4.2 ('We train to 128k context for inference at 128k')" } }, { "name": { "value": "Trinity Large", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 400, "provenance": "stated", "ref": "Abstract" }, "layers": { "value": 60, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 3072, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "value": 12288, "provenance": "stated", "ref": "Table 2 (FFN intermediate dim 12288; per-expert size 3072)" }, "attention_heads_q": { "value": 48, "provenance": "stated", "ref": "Table 2" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 2" }, "context_length": { "value": 256000, "provenance": "stated", "ref": "§3.4.2 ('We train to 256k context for inference at 512k')", "note": "trained at 256K, served at 512K" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "54/60 layers routed MoE (6 initial dense layers, Table 2 / §3.4.1) = 90%" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "Table 2 (MoE routed experts 256), §3.4.1" }, "experts_per_token": { "value": 4, "provenance": "stated", "ref": "Table 2 (Activated experts / token 4), §3.4.1" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "Table 2 (MoE shared experts 1), §2.3 ('an always-active shared expert')" }, "routing": { "value": "sigmoid routing with normalized router scores (no softmax); top-K selected by router score + expert bias, gating scores (bias excluded) renormalized and applied to each expert's output", "provenance": "stated", "ref": "§2.3 (Eq. 16-18)" }, "load_balancing": { "value": "Trinity Large: SMEBU (Soft-clamped Momentum Expert Bias Updates) — tanh soft-clamped magnitude-aware bias updates with momentum buffer (lambda=5e-4, beta=0.5, kappa=2), plus sequence-wise auxiliary loss (alpha=1e-4); Mini/Nano: standard auxiliary-loss-free sign-based bias updates with re-centering", "provenance": "stated", "ref": "§2.3 (Eq. 19-32), §6" }, "expert_granularity": { "value": "fine-grained routed experts (DeepSeekMoE design); Trinity Large opts for coarser-grained experts (expert size 3072, 4 active per token) for throughput, with greatly increased sparsity", "provenance": "stated", "ref": "§2.3, §3.4.1" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.2 (GQA + gated attention + interleaved SWA local / global attention)" }, "attention_layer_pattern": { "value": "3:1 local:global repeating for full depth: 3 local SWA layers (RoPE, window 4096) then 1 global NoPE layer; totals 45 local + 15 global (60 layers), with 6 initial dense layers", "provenance": "stated", "ref": "§2.2, Table 2" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard residual transformer (sandwich-norm residual paths, §2.4); no learned attention over prior layer outputs" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.2 (Eq. 6-7; RoPE in local layers, NoPE in global layers)" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE base frequency theta discussed qualitatively (§3.5) but no value disclosed" }, "partial_rope": { "value": false, "provenance": "inferred", "note": "Absence rule: no partial-RoPE mention; RoPE applies to local layers, NoPE to global layers" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "direct training at the target sequence length (256K for Large) from the final pretrained checkpoint — no progressive extension and no YaRN/NTK/PI; only global (NoPE) layers effectively adjusted (local SWA window and RoPE base left unchanged), which the report shows recovers loss much faster", "ref": "§3.5" }, "factor": { "value": null, "provenance": "unknown", "note": "no rescaling factor disclosed" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§3.5 ('We trained Trinity Large at a sequence length of 256K, targeting a final context window size of 256K')" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.4 (Eq. 33-36), §2.2 (QK-norm uses RMSNorm)" }, "placement": { "value": "mixed", "provenance": "stated", "ref": "§2.4 (depth-scaled sandwich norm: 'Both the input and output of the module are normalized' — pre-norm and post-norm per sublayer)" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2.2 (QK-normalization: RMSNorm applied to queries and keys before scaled dot-product attention, Eq. 4-5)" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§2.3 ('We use the SwiGLU (Shazeer, 2020) activation function as the nonlinearity')" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens; gated attention is adopted partly because prior work found it reduces (emergent) attention sinks (§2.2), which is a prose observation, not a designed sink mechanism" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no logit softcapping described" }, "other": [ { "value": "z-loss with weight 1e-6 adopted mid-training to stabilize maximum/mean logits", "provenance": "stated", "ref": "§6" }, { "value": "Depth-scaled sandwich norm: second RMSNorm gain initialized to 1/sqrt(L)", "provenance": "stated", "ref": "§2.4 (Eq. 35)" }, { "value": "Embedding activations scaled by sqrt(d) (following Takase et al. 2025)", "provenance": "stated", "ref": "§2.5 (Eq. 39)" }, { "value": "QK-norm chosen specifically because Muon training amplifies max attention logits", "provenance": "stated", "ref": "§2.2" }, { "value": "Intra-document attention masking during Trinity Large pre-training (with RSDB packing)", "provenance": "stated", "ref": "§3.2, §6" }, { "value": "Six stability fixes applied jointly after expert collapse (SMEBU, BF16 fallback from MXFP8, z-loss, sequence-wise aux loss, 3->6 dense layers, intra-doc masking); no controlled ablations", "provenance": "stated", "ref": "§6" } ] } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§3.5, §3.4.2 (Large trained at 256K sequence length)" }, "deployed_length": { "value": 524288, "provenance": "stated", "ref": "§3.4.2 ('We train to 256k context for inference at 512k')" }, "extension_method": { "value": "direct training at 256K from the final pretrained checkpoint (no progressive extension); only global (NoPE) layers adjusted while local SWA window/RoPE base stay fixed; 117B-token long-context dataset (length-biased pretraining sampling, olmOCR/FinePDF-edu OCR PDFs, regenerated ProLong at full length, FLAN/math/code); MK-NIAH @256K 0.994, @512K 0.976, @1M 0.42", "provenance": "stated", "ref": "§3.5" } }, "tokenizer": { "algorithm": { "value": "BPE", "provenance": "stated", "ref": "§2.1 ('We train a custom 200,000-token BPE vocabulary')" }, "notes": { "value": "DeepSeek-V3-inspired multi-stage pretokenizer: digit isolation with place-aligned 3-digit chunking (510-char cap avoids regex catastrophic backtracking), script-aware isolation extended to Thai/Lao/Khmer/Myanmar/Hangul, DeepSeek V3 main text regex, byte-level fallback; SuperBPE evaluated but rejected (better compression, no downstream gain); trained on ~48GB (~10B tokens) before the Large corpus was finalized, so CJK coverage trails DeepSeek V3/Qwen 3", "provenance": "stated", "ref": "§2.1.1, §2.1.2, §2.1.3" } }, "training": { "tokens": { "value": 17, "provenance": "stated", "ref": "Abstract, §1, §3.1 (Large: 17T of a 20T mix; Nano/Mini: 10T each)" }, "data_composition": { "value": "DatologyAI-curated: 20T-token mix (13T/4T/3T across three phases) combining curated web-scale data with >8T synthetic tokens (approx 6.5T rephrased web via BeyondWeb-style generation, ~1T multilingual, ~800B code); targets English plus 14 languages (Arabic, Mandarin, Japanese, Spanish, German, French, Italian, Portuguese, Indonesian, Russian, Vietnamese, Hindi, Korean, Bengali); 10T mix reuses the AFM-4.5B dataset with more math and code", "provenance": "stated", "ref": "§3.1" }, "curriculum": { "value": "three phases with midtraining mix shifts toward code/math/science and higher-quality data; on-the-fly tokenization with sequence packing; RSDB (Random Sequential Document Buffer) + BatchHet metric introduced for Trinity Large phase 3 (BatchHet reduced 4.23x, step-to-step variance 2.4x)", "provenance": "stated", "ref": "§3.1, §3.2" }, "optimizer": { "value": "Muon for hidden layers + AdamW for embedding and output layers; no RMS rescaling of Muon updates; fanout/fanin learning-rate adjustment rule lr_adj = lr * sqrt(max(1, fanout/fanin))", "provenance": "stated", "ref": "§3.4.2" }, "lr_schedule": { "value": "linear warmup of 2000 steps; Large peak LR 8.0e-4 (Muon) / 2.0e-4 (AdamW); cosine decay to 1/10 peak in decay phase; context extension continues cosine decay from 1/10 to 1/20 of peak (8.0e-5 -> 4.0e-5 Muon, 2.0e-5 -> 1.0e-5 AdamW)", "provenance": "stated", "ref": "§3.4.2" }, "batch_schedule": { "value": "Large: global batch 12288 at sequence length 8192, increased to 16384 after crossing 4.9T tokens; Nano/Mini: 4096 -> 8192 when scaling GPU clusters", "provenance": "stated", "ref": "§3.4.2" }, "precision": { "value": "BF16 (MXFP8 kernels for linear layers and grouped GEMMs were disabled mid-run with fallback to BF16 to stabilize training)", "provenance": "stated", "ref": "§6; inference benchmarks run with FP8 quantization (§5.2)" }, "parallelism": { "value": "HSDP: multiple model replicas with FSDP within replica groups (FSDP group size 128); Trinity Large additionally uses Expert Parallelism within a GPU node (EP group size 8); context parallelism degree 4 for Large context extension", "provenance": "stated", "ref": "§3.3" }, "hardware": { "value": "Trinity Large: 2048 B300 GPUs; Trinity Nano and Mini: 512 H200 GPUs (Prime Intellect clusters); modified TorchTitan framework", "provenance": "stated", "ref": "§3.3" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4 (SFT at 64K sequence length; public + custom instruction data with synthetic teacher-model instructions; heavy agentic coding supervision via OpenCode trajectories; Cut Cross-Entropy)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§4 (short RL stage using prime-rl with verifiable rewards and a learned reward-model fallback; no RLHF/DPO/GRPO named)", "note": "RL algorithm not designated in the report" }, "reasoning_training": { "value": "short RL stage on verifiable rewards (strict answer-format validation) with learned reward model for prompts without ground truth; no reasoning/CoT-specific curriculum described; report explicitly frames Trinity-Large-Preview as a light, preliminary post-training", "provenance": "stated", "ref": "§4, §7" }, "distillation": { "value": "none", "provenance": "inferred", "note": "models pre-trained from scratch (Abstract, §1); SFT data includes synthetic instructions from stronger teacher models — teacher-generated training data, which per schema convention is not distillation (noted in prose)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "no non-text modality described; all evaluations are text tasks" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": "GQA with 8 KV heads + 3:1 local/global pattern: 45/60 layers are SWA (window 4096) so their KV cache is bounded by the window; 15 global (NoPE) layers keep full cache; GQA reduces KV-cache size relative to MHA", "provenance": "derived", "note": "from attention design §2.2 (GQA head sharing Eq. 9; SWA valid positions Eq. 8)" }, "quantization": { "value": "FP8 (all inference benchmarks run with models quantized to FP8 in vLLM)", "provenance": "stated", "ref": "§5.2" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report is silent on speculative decoding" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "vLLM used for inference benchmarks (§5.2) but no serving optimizations disclosed" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MBPP+ (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 88.62, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Minerva MATH500 (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 65.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HellaSwag 5-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 90.11, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "WinoGrande 5-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 80.82, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU 5-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 82.58, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU-Pro 5-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 66.02, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "TriviaQA 5-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 83.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ARC Challenge 0-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 65.44, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BBH few-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 65.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA Diamond 5-shot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 43.94, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU (Preview)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 87.21, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Pro (Preview)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 75.25, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA Diamond (Preview)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 63.32, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SimpleQA (Preview)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 23.92, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "AIME25 (Preview)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 24.36, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Trinity family culminating in Trinity Large: 400B total / 13B active open-weight MoE with extreme sparsity, trained with zero loss spikes across all three models", "provenance": "stated", "ref": "Abstract, §7" }, "ref": { "value": "Abstract, §7", "provenance": "stated", "ref": "Abstract, §7" } }, { "text": { "value": "SMEBU (Soft-clamped Momentum Expert Bias Updates): tanh soft-clamped magnitude-aware momentum bias updates for auxiliary-loss-free MoE load balancing, addressing sign-update oscillation near convergence as expert count grows", "provenance": "stated", "ref": "§2.3" }, "ref": { "value": "§2.3", "provenance": "stated", "ref": "§2.3" } }, { "text": { "value": "RSDB (Random Sequential Document Buffer) and the BatchHet metric: reduce intra-batch document correlation in sequence packing (BatchHet -4.23x and step-to-step variance -2.4x in Trinity Large phase 3)", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } }, { "text": { "value": "Architecture synthesis: 3:1 local/global attention (RoPE local, NoPE global) + gated attention + QK-norm + depth-scaled sandwich norm + sigmoid-routed fine-grained MoE with shared expert", "provenance": "stated", "ref": "Abstract, §2" }, "ref": { "value": "Abstract, §2", "provenance": "stated", "ref": "Abstract, §2" } }, { "text": { "value": "One of the largest publicly documented synthetic-data efforts for pretraining: over 8 trillion synthetic tokens generated via DatologyAI's Ray/vLLM-on-Kubernetes stack", "provenance": "stated", "ref": "§3.1" }, "ref": { "value": "§3.1", "provenance": "stated", "ref": "§3.1" } }, { "text": { "value": "Muon optimizer at 2048-B300 scale and smooth context extension: trained at 256K for a 256K target with MK-NIAH @256K 0.994, extrapolating to 0.976 @512K and 0.42 @1M without training at those lengths", "provenance": "stated", "ref": "§3.3, §3.5" }, "ref": { "value": "§3.3, §3.5", "provenance": "stated", "ref": "§3.3, §3.5" } } ], "assessment": { "value": "The block is a well-documented synthesis of mechanisms already in the atlas rather than a new mixing family: the 3:1 local/global RoPE/NoPE schedule follows Yang et al. 2025 and the Gemma/Qwen3-Next local-global lineage; QK-norm, gated attention, sandwich norm, sigmoid routing and aux-loss-free balancing are each prior work (Henry 2020, Qiu 2025, Yin/Pangu Ultra 2025, Wang 2024a/DeepSeek-V3). Gated attention is the first appearance in the atlas. The genuinely new items are engineering-level: SMEBU (a tanh+momentum refinement of DeepSeek-V3's sign-based bias update) and RSDB/BatchHet for dataloader stability, plus the Muon-at-scale training recipe (2048 B300s, zero loss spikes). The report is candid that the six stability fixes were applied jointly without ablations. Note: the gallery card labels Trinity 'reasoning-specialised', but the report frames the family as general-purpose open-weight foundations for enterprise/agentic deployment with an emphasis on efficiency and stability; the light post-training (Preview, AIME25 24.36) does not support a reasoning-specialisation claim.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, google-gemma-3, openai-gpt-oss, zai-glm-4-5) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "54/60 layers routed MoE (6 initial dense layers, Table 2) = 90% >= 80%" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "45 local SWA + 15 global attention layers; SWA share 45/60 = 75% < 80% majority threshold (3:1 pattern, §2.2)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no multimodal content in report)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (HF checkpoints); data composition described but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "400B >= 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as general-purpose open-weight foundations for enterprise/agentic deployment (§1), with efficiency, stability and long-context (256K/512K) as design drivers. Gallery card labels it reasoning-specialised, but the report does not frame it so (no thinking-mode/CoT curriculum; Preview AIME25 24.36)." } }, "prose": { "design_overview": { "value": "Trinity Large is Arcee AI's 400B-parameter MoE (13B active per token) and the flagship of a three-model family (Nano 6B/1B, Mini 26B/3B) trained as a scaling ladder. The block is a conservative synthesis of the 2025 open-model toolkit, tuned for training stability under the Muon optimizer and for inference efficiency. Attention interleaves three local sliding-window layers (window 4096, RoPE) with one global layer per four, the global layers running without positional embeddings (NoPE); the 3:1 schedule follows the RoPE-to-NoPE hybrid results of Yang et al. and gives the 45:15 layer split. On top of GQA and QK-norm (RMSNorm on queries and keys, chosen because Muon inflates attention logits), each head's output is elementwise gated by a sigmoid projection — gated attention — which prior work ties to fewer attention sinks and fewer loss spikes. Layers use a depth-scaled sandwich norm (pre- and post-RMSNorm with the post gain initialized to 1/sqrt(L)). The MoE follows DeepSeekMoE: 256 fine-grained routed experts plus one always-active shared expert, but Trinity Large deliberately activates only 4 experts per token with larger experts (size 3072) for throughput, reaching ~30x sparsity; routing is normalized sigmoid with bias-decoupled top-K, and load balancing combines the new SMEBU scheme (tanh soft-clamped, momentum-smoothed bias updates) with a small sequence-wise auxiliary loss. The first 6 layers are dense to stabilize early representations. Training ran 17T tokens on 2048 B300 GPUs with Muon + AdamW, HSDP/FSDP with in-node expert parallelism, and DatologyAI's >8T-token synthetic data mix; context was extended by training directly at 256K with only the NoPE global layers 'adjusted', yielding MK-NIAH 0.994 at 256K and 0.976 extrapolated at 512K.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "3:1 local/global attention (RoPE local, NoPE global) hybrid schedule", "provenance": "stated", "ref": "§2.2 (cites Yang et al. 2025)" }, "origin": { "value": "Yang et al. 2025 'RoPE to NoPE' (not in atlas)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Gated attention (elementwise sigmoid gating of attention output)", "provenance": "stated", "ref": "§2.2 (cites Qiu et al. 2025)" }, "origin": { "value": "Gated Attention (Qiu et al. 2025, not in atlas)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "QK-normalization (RMSNorm on queries and keys)", "provenance": "stated", "ref": "§2.2 (cites Henry et al. 2020)" }, "origin": { "value": "QK-Norm (Henry et al. 2020)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Fine-grained routed experts with always-active shared expert (DeepSeekMoE design)", "provenance": "stated", "ref": "§2.3 (cites Dai et al. 2024)" }, "origin": { "value": "DeepSeekMoE / DeepSeek-V3", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "Sigmoid routing and auxiliary-loss-free bias-based load balancing", "provenance": "stated", "ref": "§2.3 (cites Wang et al. 2024a)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "Depth-scaled sandwich norm (pre + post normalization, depth-scaled gain)", "provenance": "stated", "ref": "§2.4 (cites Yin et al. 2025; Ding et al. 2021; Kim et al. 2025)" }, "origin": { "value": "Pangu Ultra (Yin et al. 2025, not in atlas)", "provenance": "stated", "ref": "§2.4" } }, { "mechanism": { "value": "Muon optimizer for hidden layers with AdamW for embeddings", "provenance": "stated", "ref": "§3.4.2 (cites Jordan et al. 2024)" }, "origin": { "value": "Muon (Jordan et al. 2024, not in atlas; also used by GLM-4.5)", "provenance": "stated", "ref": "§3.4.2" } }, { "mechanism": { "value": "Sequence-wise load-balance auxiliary loss", "provenance": "stated", "ref": "§2.3 (cites DeepSeek-AI et al. 2025a)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.3" } } ], "influence_out": [], "notable_omissions": { "value": [ "No license named for the weights release", "No compute disclosed (FLOPs/GPU-hours)", "RL algorithm not named (prime-rl; no PPO/GRPO/DPO designation)", "Post-training deliberately light: Trinity-Large-Preview is a 'preliminary release' with only 5 instruct benchmarks; no agentic or long-context evaluation tables for the chat model", "No per-benchmark comparison table vs peers (Figure 3 is chart-only)", "RoPE base frequencies not disclosed (theta discussed qualitatively)", "The six stability fixes were applied jointly with no controlled ablations (admitted in §6)", "Embedding/output-head tying not disclosed", "MK-NIAH long-context results appear only in the context-extension section, not the main evaluation suite", "Tokenizer trained before the Large corpus was finalized, so CJK compression trails peers (stated)", "No training loss curves for Nano/Mini (Figure 1 covers Large only)", "SFT data recipe described qualitatively; no data volumes for post-training" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "arcee-ai-trinity-large-400b.json" }, { "slug": { "value": "cisco-antares-1b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://cisco-foundation-ai.github.io/antares/technical-report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/cisco-antares-1b", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Antares", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "350M", "provenance": "stated", "ref": "Abstract / eval tables" }, { "value": "1B", "provenance": "stated", "ref": "Abstract / eval tables" }, { "value": "3B", "provenance": "stated", "ref": "eval tables" } ], "org": { "value": "Cisco Foundation AI", "provenance": "inferred", "note": "Report byline / cisco-foundation-ai.github.io" }, "release_date": { "provenance": "unknown", "note": "Undated non-arXiv PDF; no announcement dates used" }, "report_url": { "value": "https://cisco-foundation-ai.github.io/antares/technical-report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "provenance": "unknown" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'publicly release Antares-350M and Antares-1B'" } }, "scale": { "total_params": { "value": 1, "provenance": "stated", "ref": "name convention (Antares-1B; report leads with 350M and 1B releases)" }, "active_params_per_token": { "value": 1, "provenance": "derived", "note": "Dense model: active = total (1B reference)" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "ffn_ratio": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "head_dim": { "provenance": "unknown" }, "vocab_size": { "provenance": "unknown" }, "embedding_tied": { "value": true, "provenance": "stated", "ref": "architecture blueprint (tied embeddings)" }, "variants": [ { "name": { "value": "350M", "provenance": "stated", "ref": "Abstract" }, "total_params": { "value": 0.35, "provenance": "stated", "ref": "name convention" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 32768, "provenance": "stated", "ref": "context section" } }, { "name": { "value": "1B", "provenance": "stated", "ref": "Abstract" }, "total_params": { "value": 1, "provenance": "stated", "ref": "name convention" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "context section" } }, { "name": { "value": "3B", "provenance": "stated", "ref": "eval tables" }, "total_params": { "value": 3, "provenance": "stated", "ref": "name convention" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "context section" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "inferred", "note": "Fine-tuned from IBM Granite 4.0 checkpoints (dense family); no MoE described" }, "moe": { "expert_count": { "provenance": "n/a", "ref": "architecture blueprint (dense)" }, "experts_per_token": { "provenance": "n/a", "ref": "architecture blueprint (dense)" }, "shared_experts": { "provenance": "n/a", "ref": "architecture blueprint (dense)" }, "routing": { "provenance": "n/a", "ref": "architecture blueprint (dense)" }, "load_balancing": { "provenance": "n/a", "ref": "architecture blueprint (dense)" }, "expert_granularity": { "provenance": "n/a", "ref": "architecture blueprint (dense)" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "architecture blueprint (Granite 4.0: GQA)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "Absence rule: no per-layer pattern described" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "architecture blueprint" }, "rope_base": { "provenance": "unknown" }, "partial_rope": { "provenance": "unknown" }, "extension": { "method": { "provenance": "unknown" }, "factor": { "provenance": "unknown" }, "target_length": { "provenance": "unknown" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "architecture blueprint" }, "placement": { "provenance": "unknown" }, "qk_norm": { "provenance": "unknown" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "architecture blueprint" }, "stability": { "attention_sinks": { "provenance": "unknown", "note": "Blueprint mentions no stability tricks; not detailed enough for the absence rule" }, "softcapping": { "provenance": "unknown" }, "other": { "provenance": "unknown" } } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "context section (1B variant: 128K)" }, "deployed_length": { "value": 131072, "provenance": "inferred", "note": "Report discloses a single context length per variant" }, "extension_method": { "provenance": "unknown" } }, "tokenizer": { "algorithm": { "provenance": "unknown" }, "notes": { "provenance": "unknown" } }, "training": { "tokens": { "provenance": "unknown" }, "data_composition": { "provenance": "unknown" }, "curriculum": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "optimizer": { "value": "AdamW", "provenance": "stated", "ref": "training section" }, "lr_schedule": { "value": "Cosine, peak 5e-5", "provenance": "stated", "ref": "training section" }, "batch_schedule": { "provenance": "unknown" }, "precision": { "provenance": "unknown" }, "parallelism": { "value": "FSDP", "provenance": "stated", "ref": "training section" }, "hardware": { "value": "Single 8×H100 node", "provenance": "stated", "ref": "training section" }, "compute": { "provenance": "unknown" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "post-training section (SFT, 1 epoch, semantic-conditioning auxiliary objective)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "post-training section (verifiable multi-component rewards, no reward model, KL vs SFT reference, veRL+vLLM)" }, "reasoning_training": { "value": "GRPO with verifiable multi-component rewards over terminal vulnerability-localization trajectories", "provenance": "stated", "ref": "post-training section" }, "distillation": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes code/terminal text tasks only" }, "attachment": { "provenance": "n/a", "ref": "model description" } }, "efficiency": { "kv_cache": { "provenance": "unknown", "note": "Report does not discuss KV cache" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "evaluation": { "benchmarks": [ { "name": { "value": "VLoc Bench File F1 (Antares-1B)", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 0.209, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "VLoc Bench File F1 (Antares-3B)", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 0.223, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "VLoc Bench File F1 (Antares-350M)", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 0.135, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "VLoc Bench File F1 (GPT-5.5 xhigh)", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 0.229, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Agentic vulnerability localization with open 350M/1B models", "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "text": { "value": "Semantic-conditioning auxiliary objective during SFT", "provenance": "stated", "ref": "post-training section" }, "ref": { "value": "post-training section", "provenance": "stated", "ref": "post-training section" } }, { "text": { "value": "GRPO with verifiable multi-component rewards, no reward model", "provenance": "stated", "ref": "post-training section" }, "ref": { "value": "post-training section", "provenance": "stated", "ref": "post-training section" } } ], "assessment": { "value": "Architecture novelty ≈ 0 — Antares is an SFT+GRPO post-training of IBM Granite 4.0 checkpoints (the block section records only the Granite blueprint names, everything else unknown). The contribution is the recipe: terminal-trajectory fine-tuning with semantic conditioning and verifiable-reward GRPO for security tasks. The atlas's thinnest architecture record, honestly so.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "from block: Granite 4.0 dense blueprint" }, "sequence_mixing": { "value": "full-attention", "provenance": "inferred", "note": "Granite 4.0 blueprint describes GQA full attention; no efficient-attention mechanisms disclosed" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data not released" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "1B reference variant" }, "design_intent": { "value": "domain-specific", "provenance": "inferred", "note": "framed for agentic vulnerability localization (Abstract)" } }, "prose": { "design_overview": { "value": "Antares is not a new architecture: it is a post-training recipe applied to IBM Granite 4.0 checkpoints at 350M, 1B, and 3B, released openly for agentic vulnerability localization. The report states the inherited blueprint — GQA attention, SwiGLU, RMSNorm, RoPE, tied embeddings — but discloses no layer counts, dimensions, or head numbers, so the scale section records only what the report says (sizes, context lengths of 32K/128K/128K). The contribution is entirely in the recipe: one epoch of SFT with a semantic-conditioning auxiliary objective on terminal trajectories, then GRPO with verifiable multi-component rewards (no reward model) using veRL+vLLM under FSDP on a single 8×H100 node, with KL regularization against the SFT reference. Evaluation is on the in-house VLoc Bench file-F1 metric, where Antares-1B reaches 0.209 against 0.229 for a much larger frontier model.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Base checkpoints (350M/1B/3B from Granite 4.0)", "provenance": "stated", "ref": "Abstract / model section" }, "origin": { "value": "IBM Granite 4.0", "provenance": "stated", "ref": "Abstract" } }, { "mechanism": { "value": "GRPO with verifiable rewards", "provenance": "stated", "ref": "post-training section" }, "origin": { "value": "GRPO line (DeepSeekMath et al.)", "provenance": "inferred", "note": "report cites GRPO; origin naming is analyst judgement" } } ], "influence_out": [], "notable_omissions": { "value": [ "No architecture table: layers, hidden dims, heads, vocab, tokenizer all undisclosed", "No data mixture or token counts", "No license named", "Release date unknown (undated PDF, not on arXiv)", "Evaluation limited to the in-house VLoc Bench" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "cisco-antares-1b.json" }, { "slug": { "value": "coherelabs-tiny-aya-3-35b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2603.11510", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/coherelabs-tiny-aya-3-35b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Tiny Aya", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "Tiny Aya Base", "provenance": "stated", "ref": "Abstract, §1" }, { "value": "Tiny Aya Global", "provenance": "stated", "ref": "Abstract, §1" }, { "value": "Tiny Aya Earth", "provenance": "stated", "ref": "Abstract, §1" }, { "value": "Tiny Aya Fire", "provenance": "stated", "ref": "Abstract, §1" }, { "value": "Tiny Aya Water", "provenance": "stated", "ref": "Abstract, §1" } ], "org": { "value": "Cohere Labs / Cohere", "provenance": "stated", "ref": "Author affiliations (Cohere Labs; Cohere)" }, "release_date": { "value": "2026-03-12", "provenance": "stated", "ref": "arXiv stamp 'arXiv:2603.11510v1 [cs.CL] 12 Mar 2026'" }, "report_url": { "value": "https://arxiv.org/abs/2603.11510", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "model license not named; the only license appearing is the arXiv paper license (CC BY-NC-ND 4.0, arXiv page header)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "§1 ('a family of efficient, open-weight multilingual models'), §7 ('We release a family of 3.35B-parameter models: Tiny Aya Base, Tiny Aya Global, and region specific models Tiny Aya Water, Tiny Aya Earth, Tiny Aya Fire')" } }, "scale": { "total_params": { "value": 3.35, "provenance": "stated", "ref": "Table 2 (Total parameters 3.35B)" }, "active_params_per_token": { "value": 3.35, "provenance": "derived", "note": "dense decoder-only Transformer (stated §3.1): all parameters active per token, so active = total = 3.35B" }, "layers": { "value": 36, "provenance": "stated", "ref": "Table 2 (Num layers 36)" }, "hidden_dim": { "value": 2048, "provenance": "stated", "ref": "Table 2 (Embedding dims 2048)" }, "ffn_inner_dim": { "value": 11008, "provenance": "stated", "ref": "Table 2 (FFN hidden dims 11008)" }, "ffn_ratio": { "value": 5.375, "provenance": "derived", "note": "11008 / 2048 = 5.375 (Table 2 values)" }, "attention_heads_q": { "value": 16, "provenance": "stated", "ref": "Table 2 (Num heads 16)" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "Table 2 (Num KV heads 4), §3.1 (grouped-query attention)" }, "head_dim": { "value": 128, "provenance": "derived", "note": "2048 / 16 heads = 128 (Table 2 values)" }, "vocab_size": { "value": 262144, "provenance": "stated", "ref": "§2.1 ('a vocabulary size of 262k'), Table 2 (Vocab size 262k)" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed (Table 2 breaks out Embedding parameters 0.5B vs Non-Embedding 2.8B but does not state tying)" }, "variants": [] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "stated dense decoder-only Transformer (§3.1) with no MoE components; 3.35B total = 3.35B active" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§3.1 (dense decoder-only Transformer)" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§3.1 (dense decoder-only Transformer)" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§3.1 (dense decoder-only Transformer)" }, "routing": { "value": null, "provenance": "n/a", "ref": "§3.1 (dense decoder-only Transformer)" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§3.1 (dense decoder-only Transformer)" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§3.1 (dense decoder-only Transformer)" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§3.1 (interleaved sliding-window and full attention layers in a 3:1 ratio), Table 2 (Sliding window 4096)" }, "attention_layer_pattern": { "value": "3:1 interleave of sliding-window attention (4096-token window) and full attention layers across the 36 layers, following Command A; sliding-window layers use RoPE, full-attention layers use NoPE", "provenance": "stated", "ref": "§3.1" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no attention over prior layer outputs described; standard parallel residual Transformer blocks (attention and FFN computed in parallel) stated §3.1" }, "position_encoding": { "method": { "value": "other", "provenance": "stated", "ref": "§3.1 (SWA layers use RoPE; full-attention layers use NoPE)", "note": "mixed scheme with no single enum value: 3:1 sliding-window (RoPE) to full-attention (NoPE) interleave, following Command A" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE base frequency not disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "RoPE is applied to the sliding-window layers only (per-layer split), but per-dimension partial RoPE is not described" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "no context extension mechanism described; Table 2 fixes input/output context at 8192 tokens" }, "factor": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "target_length": { "value": null, "provenance": "unknown", "note": "no extension target disclosed" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "only generic 'layer norms' mentioned in the FP32-precision list (§3.1); the norm family (RMSNorm vs LayerNorm) is not named" }, "placement": { "value": null, "provenance": "unknown", "note": "not described (parallel-block layout stated, norm placement not)" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: attention described via GQA and the SWA/NoPE interleave (§3.1) with no QK-normalisation mention" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§3.1 (SwiGLU activations, no bias in dense layers)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no attention-logit softcapping described" }, "other": { "value": null, "provenance": "unknown", "note": "no other stability mechanisms disclosed (bias-free dense layers §3.1 is a parameterization choice, not a stability mechanism)" } } }, "context": { "trained_length": { "value": 8192, "provenance": "stated", "ref": "Table 2 (Input Context (tokens) 8192)" }, "deployed_length": { "value": 8192, "provenance": "stated", "ref": "Table 2 (Input Context (tokens) 8192, Output Context (tokens) 8192)" }, "extension_method": { "value": null, "provenance": "unknown", "note": "no extension mechanism described; the report does not state that training ran at the full deployed length" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "subword algorithm not named in the report (pre-tokenization uses the GPT-4o regex with no normalization; details deferred to Abagyan et al. 2025)" }, "notes": { "value": "single massively multilingual tokenizer shared by all models; 262k vocabulary trained on 50GB of Fineweb-2 sampled with a weighting that combines data-distribution weight and language-family/script bucket weight (wi = wid·wib / Σ); GPT-4o regex pre-tokenization, no normalization; competitive or superior tokens-per-character compression across scripts, especially Khmer, Telugu, Gujarati, Lao, Ge'ez", "provenance": "stated", "ref": "§2.1" } }, "training": { "tokens": { "value": 6, "provenance": "stated", "ref": "§3.1 ('We pretrain Tiny Aya model for 6T tokens')" }, "data_composition": { "value": "large corpus of public and proprietary sources covering 70 languages plus programming-language datasets; filtering pipeline: (1) language ID and stopword filtering, (2) heuristic cleaning from raw sources, (3) deduplication, (4) domain classification and quality filtering; cooldown (mid-training) mixture upsampling the highest-quality pretraining datasets plus instruction-style datasets spanning all 70 languages", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "Warmup-Stable-Decay (WSD) schedule with a cooldown (mid-training) mixture following SmolLM3-3B; hyperparameters chosen via 200B-token ablation runs with 40B-token cooldowns", "provenance": "stated", "ref": "§3.1, §2.2" }, "optimizer": { "value": null, "provenance": "unknown", "note": "optimizer not named (only 'optimizer states' kept in FP32, §3.1)" }, "lr_schedule": { "value": "pretraining: Warmup-Stable-Decay (WSD); posttraining (all cluster models): cosine decay, peak 2.5e-5, final 1.2e-6", "provenance": "stated", "ref": "§3.1, §3.2" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "pretraining batch size not disclosed; posttraining uses a global batch size of 32 (§3.2)" }, "precision": { "value": "FP8 training mixing FP8, BF16 and FP32: main weights and optimizer states in FP32, weights cast to BF16/FP8 before computation; exponentials, softmaxes, layer norms and output embeddings in FP32; attention computation in BF16", "provenance": "stated", "ref": "§3.1" }, "parallelism": { "value": null, "provenance": "unknown", "note": "pretraining runs on the JAX-based Fax framework (stated §3.1) but the parallelization strategy is not described" }, "hardware": { "value": "256 NVIDIA H100 GPUs for pretraining; 16 NVIDIA H100 GPUs per cluster model for posttraining (each cluster run completes within 24 hours wall-clock)", "provenance": "stated", "ref": "§3.1, §3.2" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§2.3, §3.2 (region-specific SFT on five regional data subsets plus a global SFT model over all regions; 3 epochs, cosine decay, global batch 32)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§3.2 ('a minimal preference tuning phase on top of SFT for the Tiny Aya Global model... teaches the model its identity... while maintaining multilingual safety')", "note": "a preference-tuning phase exists but no algorithm (RLHF/DPO/GRPO) is named" }, "reasoning_training": { "value": "none disclosed", "provenance": "inferred", "note": "no reasoning-RL stage described; chain-of-thought is used only at evaluation time (§5.4.1)" }, "distillation": { "value": "none", "provenance": "inferred", "note": "model is pretrained from scratch (6T tokens, §3.1); FusioNN teachers (Gemma3-27B-It, Command A, DeepSeek-V3) generate SFT completions only (§2.3.1), which per schema is teacher-generated training data, not distillation — noted in prose" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "no non-text modality described; all training data, tasks, and benchmarks (translation, MMLU, PIQA, Dolly, MGSM, MultiJail, NormAd, BLEnD) are text-based" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1, §4 (text-only model)" } }, "efficiency": { "kv_cache": { "value": "GQA with 16 query heads over 4 KV heads: KV-cache footprint proportional to 4 heads (1/4 of MHA); sliding-window layers need only the 4096-token window cached", "provenance": "derived", "note": "derived from stated attention config (Table 2, §3.1); the report itself does not quantify KV cache" }, "quantization": { "value": "llama.cpp formats q4_0, q4_k_m and q8_0, served via llama.cpp and MLX; Q4_K_M optimal: 2.14 GB memory, 32.4 tokens/s decode, ~1.4-point mDolly degradation (Q4_0: ~2.1 points; Q8_0: negligible)", "provenance": "stated", "ref": "§6" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "§6 (no speculative decoding mentioned)" }, "serving": { "value": "llama.cpp and MLX inference stacks for edge deployment; ~10 tokens/s decode on iPhone 13 and ~32 tokens/s on iPhone 17 Pro (Q4_K_M); low prefill throughput on older devices raises TTFT", "provenance": "stated", "ref": "§6" } }, "evaluation": { "benchmarks": [ { "name": { "value": "Global MMLU", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 44.9, "provenance": "stated", "ref": "Table 4 (Tiny Aya Global, avg accuracy ±7.3 over 42 languages)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "INCLUDE", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 45.1, "provenance": "stated", "ref": "Table 4 (Tiny Aya Global, ±11.1 over 44 languages)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Global PIQA", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 68.3, "provenance": "stated", "ref": "Table 4 (Tiny Aya Global, ±10.6 over 116 languages)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "mDolly (66 languages)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 86.9, "provenance": "stated", "ref": "Table 5 (Tiny Aya Global, rubric judge score; std 6.2)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "mArenaHard (66 languages)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 67.4, "provenance": "stated", "ref": "Table 5 (Tiny Aya Global, std 6.1)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "GlobalMGSM (35 languages)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 52.8, "provenance": "stated", "ref": "Table 5 (Tiny Aya Global, std 11.7)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Flores (66 languages, ChrF)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 43.5, "provenance": "stated", "ref": "Table 5 (Tiny Aya Global, std 14.0; English-to-target)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "WMT24++ (55 languages, ChrF)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 46.0, "provenance": "stated", "ref": "Table 5 (Tiny Aya Global, std 10.4; wins in 46/55 languages vs Gemma3-4B, §5.2)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MultiJail Mean Safe Rate", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 91.1, "provenance": "stated", "ref": "Table 7 (Tiny Aya Global, 10 languages)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "MultiJail Min Safe Rate", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 87.0, "provenance": "stated", "ref": "Table 7 (Tiny Aya Global)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "XSTest Over-Refusal", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 10.4, "provenance": "stated", "ref": "Table 7 (Tiny Aya Global, English)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "XSTest Under-Refusal", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 15.5, "provenance": "stated", "ref": "Table 7 (Tiny Aya Global, English)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Tiny Aya redefines what a small multilingual model can achieve: 3.35B parameters across 70 languages with state-of-the-art translation quality (best average on Flores and WMT24++, beating Gemma3-4B in 46/55 WMT24++ languages) and the highest multilingual safety (91.1% mean MultiJail safe rate) while drastically reducing language disparities", "provenance": "stated", "ref": "Abstract, §1, §5" }, "ref": { "value": "Abstract, §1, §5", "provenance": "stated", "ref": "Abstract, §1, §5" } }, { "text": { "value": "A balanced multilingual data mixture: tokenizer data weighted by language-family/script buckets (Abagyan et al. 2025 recipe) plus pretraining/posttraining mixtures that explicitly balance regional coverage, with prompt-level transformations (Naturalness, Cultural Adaptation, Difficulty Enhancement) to reduce translationese and English-centric framing", "provenance": "stated", "ref": "§2.1, §2.2, §2.3.1" }, "ref": { "value": "§2.1, §2.2, §2.3.1", "provenance": "stated", "ref": "§2.1, §2.2, §2.3.1" } }, { "text": { "value": "Region-aware posttraining with cluster-based SFT and SimMerge-guided checkpoint merging: region-specialized models (Earth, Fire, Water) merged with the Global model improve translation by up to +5.5 ChrF (South Asia) and +1.7 on average (Africa) while restoring global instruction-following and safety", "provenance": "stated", "ref": "§2.3, §3.2, §3.3, §5.2" }, "ref": { "value": "§2.3, §3.2, §3.3, §5.2", "provenance": "stated", "ref": "§2.3, §3.2, §3.3, §5.2" } }, { "text": { "value": "FusioNN (Fusion-of-NN) teacher aggregation for massively multilingual synthetic completions, with Command A as Fusor and per-language teacher selection", "provenance": "stated", "ref": "§2.3.1" }, "ref": { "value": "§2.3.1", "provenance": "stated", "ref": "§2.3.1" } }, { "text": { "value": "A comprehensive multilingual evaluation framework: rubric-based absolute ratings in lieu of win rates, language-confusion tracking, and mean-plus-minimum safety reporting to surface cross-language disparities", "provenance": "stated", "ref": "§4.3" }, "ref": { "value": "§4.3", "provenance": "stated", "ref": "§4.3" } }, { "text": { "value": "Practical edge deployment: standard llama.cpp/MLX quantization (q4_0, q4_k_m, q8_0) achieving ~10 tokens/s decode on a four-year-old iPhone 13 and 32 tokens/s on iPhone 17 Pro", "provenance": "stated", "ref": "§6" }, "ref": { "value": "§6", "provenance": "stated", "ref": "§6" } } ], "assessment": { "value": "Architecturally a faithful, smaller Command A: parallel blocks, 3:1 sliding-window/full-attention interleave with the same RoPE-on-SWA / NoPE-on-full split, GQA, SwiGLU, bias-free — the report states this lineage explicitly, so the block itself contributes no new mechanism to the atlas. The novelty is entirely in the data and training system: the language-family/script-bucket tokenizer weighting (from Cohere's own prior work), region-clustered SFT with SimMerge-guided merging into Earth/Fire/Water variants, FusioNN teacher aggregation, and the rubric/language-confusion evaluation methodology. vs google-gemma-3 (the closest atlas peer at similar scale): Tiny Aya trades a uniform attention block for the SWA/NoPE hybrid, and its claim to distinctiveness is balanced cross-language performance and safety rather than raw peaks — consistent with its own framing that it does not 'nominally score the highest' on discriminative tasks. The family's on-device framing (quantized MLX/llama.cpp deployment) is also more explicit than most atlas peers at this scale.", "provenance": "inferred", "note": "comparison against atlas entries (google-gemma-3, allenai-olmo-2-7b, microsoft-phi-3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "dense decoder-only Transformer stated (§3.1); 3.35B total = 3.35B active" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "3:1 sliding-window (RoPE) to full-attention (NoPE) interleave (§3.1): SWA share 27/36 = 75% < 80% and full-attention share 25% < 80%, so neither family dominates" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no non-text modality described)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true ('open-weight multilingual models', release of all five models); data composition described in detail (§2.1–§2.3, Appendix A) but the corpora themselves are not released" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "3.35B is within 1–10B" }, "design_intent": { "value": "on-device", "provenance": "inferred", "note": "framed as an alternative scaling path 'centered on efficiency, balanced performance across languages, and practical deployment' (Abstract), with a dedicated section on edge deployment (quantization, iPhone throughput, §6). Secondary intent: balanced massively-multilingual coverage, noted in prose" } }, "prose": { "design_overview": { "value": "Tiny Aya is a 3.35B dense decoder-only Transformer (36 layers, hidden 2048, 16 Q / 4 KV heads, SwiGLU, bias-free, parallel attention+FFN blocks) whose block is deliberately carried over from Cohere's Command A: a 3:1 interleave of sliding-window attention (4096-token window, RoPE) and full attention (NoPE). The model's real design center of gravity is the data system around that block. A single 262k-vocabulary tokenizer is shared by all five releases, trained on 50GB of Fineweb-2 with a weighting that multiplies data-distribution weight by language-family/script bucket weight so that underrepresented scripts (Khmer, Telugu, Gujarati, Ge'ez) get competitive compression. Pretraining runs 6T tokens over 70 languages plus code, with a WSD schedule and a SmolLM3-style cooldown mixture. Posttraining is region-aware: five regional SFT clusters (Europe, West Asia, South Asia, Asia-Pacific, Africa) plus a global SFT model, fed by translated and FusioNN-aggregated synthetic data from Gemma3-27B-It, Command A and DeepSeek-V3 teachers, then each region-specialized checkpoint is merged with the global model under SimMerge operator selection — producing Tiny Aya Base, Global, Earth, Fire and Water from the same weights. The report's evaluation apparatus is as much a contribution as the model: rubric-based absolute judge ratings instead of win rates, language-confusion tracking, and mean+minimum safety reporting across languages. At 8K context it targets balanced multilingual quality rather than peaks, and its edge-deployment story (q4_k_m at 2.14 GB, ~10–32 tok/s on iPhones) is the closest the atlas has to an explicitly on-device massively-multilingual model.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Parallel Transformer blocks, 3:1 sliding-window/full attention interleave, RoPE-on-SWA / NoPE-on-full split, GQA, SwiGLU, bias-free dense layers — 'closely follows the core design choices from Command A'", "provenance": "stated", "ref": "§3.1 (cites Command A / Cohere et al. 2025)" }, "origin": { "value": "Command A", "provenance": "stated", "ref": "§3.1" } }, { "mechanism": { "value": "Cooldown (mid-training) mixture upsampling high-quality and instruction-style data", "provenance": "stated", "ref": "§2.2 ('Similar to SmolLM3-3B, we use a cooldown (mid-training) mixture')" }, "origin": { "value": "SmolLM3-3B", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Multilingual tokenizer data weighting by language-family/script buckets", "provenance": "stated", "ref": "§2.1 (follows Abagyan et al. 2025)" }, "origin": { "value": "Cohere Labs tokenizer work (not in atlas)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Aya family lineage (Aya 23, Aya Expanse) as the multilingual-research context for the release", "provenance": "stated", "ref": "§7 (related work cites Aya 23 and Aya Expanse)" }, "origin": { "value": "Aya 23 / Aya Expanse (not in atlas)", "provenance": "stated", "ref": "§7" } } ], "influence_out": [], "notable_omissions": { "value": [ "Optimizer not named (only 'optimizer states in FP32')", "Pretraining batch size and peak learning-rate values not disclosed (posttraining values are given)", "Tokenizer subword algorithm not named (GPT-4o-regex pre-tokenization and Fineweb-2 data described)", "Normalization family (RMSNorm vs LayerNorm) and norm placement not stated", "Model license not named (only the arXiv paper license CC BY-NC-ND 4.0 appears)", "RoPE base frequency and per-dimension partial-RoPE not disclosed", "No per-language pretraining data proportions (only posttraining cluster mixes in Appendix A, plus English shares in Table 8)", "No context extension mechanism (fixed 8K per Table 2) and no statement that training ran at deployed length", "No training compute (FLOPs) and no parallelism strategy disclosed", "Embedding tying not disclosed", "No speculative decoding or KV-cache engineering discussion" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "coherelabs-tiny-aya-3-35b.json" }, { "slug": { "value": "deepseek-r1", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2501.12948", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/deepseek-r1", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "DeepSeek-R1", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "DeepSeek-AI", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2025-01-22", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date per export.arxiv.org API (arXiv:2501.12948)" }, "report_url": { "value": "https://arxiv.org/abs/2501.12948", "provenance": "stated", "ref": "pipeline" }, "license": { "provenance": "unknown" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'we open-source DeepSeek-R1-Zero, DeepSeek-R1, and six dense models'" } }, "scale": { "total_params": { "value": 671, "provenance": "stated", "ref": "Table 4 (# Total Params 671B)" }, "active_params_per_token": { "value": 37, "provenance": "stated", "ref": "Table 4 (# Activated Params 37B)" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "ffn_ratio": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "head_dim": { "provenance": "unknown" }, "vocab_size": { "provenance": "unknown" }, "embedding_tied": { "provenance": "unknown" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "Table 4 (Architecture: MoE; no per-layer pattern disclosed in this report)" }, "moe": { "expert_count": { "provenance": "unknown" }, "experts_per_token": { "provenance": "unknown" }, "shared_experts": { "provenance": "unknown" }, "routing": { "provenance": "unknown" }, "load_balancing": { "provenance": "unknown" }, "expert_granularity": { "provenance": "unknown" } }, "attention_variant": { "provenance": "unknown" }, "attention_layer_pattern": { "provenance": "unknown", "note": "Report does not describe the block; it is a post-training paper" }, "depth_mixing": { "provenance": "unknown", "note": "Report does not describe the block" }, "position_encoding": { "method": { "provenance": "unknown" }, "rope_base": { "provenance": "unknown" }, "partial_rope": { "provenance": "unknown" }, "extension": { "method": { "provenance": "unknown" }, "factor": { "provenance": "unknown" }, "target_length": { "provenance": "unknown" } } }, "normalization": { "type": { "provenance": "unknown" }, "placement": { "provenance": "unknown" }, "qk_norm": { "provenance": "unknown", "note": "Report does not describe the block" } }, "activation": { "provenance": "unknown" }, "stability": { "attention_sinks": { "provenance": "unknown", "note": "Report does not describe the block" }, "softcapping": { "provenance": "unknown", "note": "Report does not describe the block" }, "other": { "provenance": "unknown", "note": "Report does not describe the block" } } }, "context": { "trained_length": { "provenance": "unknown" }, "deployed_length": { "provenance": "unknown" }, "extension_method": { "provenance": "unknown", "note": "Report states a max generation length of 32,768 tokens (§2.3, inference setting), not a context length" } }, "tokenizer": { "algorithm": { "provenance": "unknown" }, "notes": { "provenance": "unknown" } }, "training": { "tokens": { "provenance": "unknown", "note": "Report is a post-training paper; no pretraining details disclosed" }, "data_composition": { "provenance": "unknown", "note": "No pretraining mixture disclosed; post-training data: cold-start CoT examples ('thousands'), rejection-sampled ~800k samples (600k reasoning + 200k non-reasoning) (§2.3)" }, "curriculum": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "optimizer": { "provenance": "unknown", "note": "GRPO is the RL algorithm (§2.2), not a pretraining optimizer" }, "lr_schedule": { "provenance": "unknown" }, "batch_schedule": { "provenance": "unknown" }, "precision": { "provenance": "unknown" }, "parallelism": { "provenance": "unknown" }, "hardware": { "provenance": "unknown" }, "compute": { "provenance": "unknown" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§2.3 (cold-start CoT SFT on DeepSeek-V3-Base; rejection-sampled SFT)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§2.2 (reasoning RL uses GRPO with rule-based rewards; second RL stage uses rule + reward-model rewards)" }, "reasoning_training": { "value": "Two-stage: (1) cold-start CoT SFT on V3-Base → GRPO reasoning RL with rule-based accuracy/format rewards (R1-Zero: pure RL from V3-Base, no SFT); (2) rejection-sampled SFT (~800k samples) → RL for all scenarios; / template; max generation 32,768 tokens; eval temp 0.6, top-p 0.95", "provenance": "stated", "ref": "§2.2–2.3" }, "distillation": { "value": "none", "provenance": "inferred", "note": "R1/R1-Zero are trained via RL from DeepSeek-V3-Base, not distilled; the report uses R1 as TEACHER to distill six dense models (§2.4) — that is this model's influence, not its origin" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes text reasoning only" }, "attachment": { "provenance": "n/a", "ref": "§2 model description" } }, "efficiency": { "kv_cache": { "provenance": "unknown", "note": "Report does not discuss KV cache" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "evaluation": { "benchmarks": [ { "name": { "value": "AIME 2024 (Pass@1)", "provenance": "stated", "ref": "Table 3 / §3.1" }, "value": { "value": 79.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MATH-500 (Pass@1)", "provenance": "stated", "ref": "Table 3 / §3.1" }, "value": { "value": 97.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Codeforces (rating)", "provenance": "stated", "ref": "Table 3 / §3.2" }, "value": { "value": 2029, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 90.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 84.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA Diamond", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 71.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AlpacaEval2 (LC)", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 87.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ArenaHard", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 92.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE Verified", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 49.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LiveCodeBench", "provenance": "stated", "ref": "Table 3 / §3.3" }, "value": { "value": 65.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "R1-Zero: first demonstration of pure RL (no SFT) eliciting strong reasoning, including the emergent 'aha moment'", "provenance": "stated", "ref": "Abstract / §2.1 / §3.1" }, "ref": { "value": "Abstract / §2.1 / §3.1", "provenance": "stated", "ref": "Abstract / §2.1 / §3.1" } }, { "text": { "value": "GRPO-based RLVR with rule-based rewards avoiding reward-model bottlenecks", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "Distilling reasoning capabilities into small dense models (1.5B–70B) with strong results", "provenance": "stated", "ref": "§2.4 / §3.4" }, "ref": { "value": "§2.4 / §3.4", "provenance": "stated", "ref": "§2.4 / §3.4" } }, { "text": { "value": "Open-sourcing R1-Zero, R1, and six distilled dense models", "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } } ], "assessment": { "value": "Architecturally this record adds nothing new to the atlas — the report states the base is DeepSeek-V3 (already a record) and discloses no block details. Its value is as the reasoning-specialised exemplar of the sparse-MoE line: the first major open report showing rule-based RLVR alone (without SFT) can elicit chain-of-thought reasoning, plus the distillation-of-reasoning pattern that several later atlas entries adopt.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "stated", "ref": "Table 4 (Architecture: MoE)" }, "sequence_mixing": { "value": "full-attention", "provenance": "inferred", "note": "Report states the model is built on DeepSeek-V3-Base (§1); per the deepseek-v3 atlas record, V3 uses full-attention MLA. Judgement call: the R1 report itself discloses no attention pattern — flagged in prose" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "671B total params" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "framed entirely around reasoning RL (Abstract/§1)" } }, "prose": { "design_overview": { "value": "DeepSeek-R1 is not an architecture paper: it is a post-training report on a 671B/37B-active MoE model whose base is DeepSeek-V3, already in the atlas. Everything the report discloses about the block fits in Table 4 — 'Architecture: MoE', 671B total, 37B activated — so every deeper spec (attention, experts, context, vocabulary) is deliberately unknown here rather than imported from the V3 record. The substance is the two-stage reasoning recipe: R1-Zero showed pure GRPO with rule-based rewards, no SFT at all, elicits long chain-of-thought and the emergent 'aha moment'; R1 adds a cold-start CoT SFT before RL, then a rejection-sampled SFT round and a second RL stage for helpfulness/harmlessness. The report also established the pattern of distilling reasoning into small dense models, which several later open-weight families in the gallery follow.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "MoE base architecture (671B total / 37B activated)", "provenance": "stated", "ref": "§1 / Table 4" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§1" } }, { "mechanism": { "value": "GRPO reinforcement learning", "provenance": "stated", "ref": "§2.2 (cites Shao et al., DeepSeekMath)" }, "origin": { "value": "DeepSeekMath (Shao et al.)", "provenance": "stated", "ref": "§2.2" } } ], "influence_out": [ { "model_slug": { "value": "deepseek-v3-2", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Group Relative Policy Optimization (GRPO) as the RL training algorithm", "provenance": "inferred", "note": "mechanism as stated in deepseek-v3-2 report's lineage_in" } }, { "model_slug": { "value": "nanbeige-4-1-3b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Group Relative Policy Optimization (GRPO)", "provenance": "inferred", "note": "mechanism as stated in nanbeige-4-1-3b report's lineage_in" } } ], "notable_omissions": { "value": [ "No block-level disclosures beyond MoE / 671B / 37B (Table 4) — attention, experts, context, vocab, tokenizer all absent", "No pretraining details (report is post-training only)", "No license named", "No compute/GPU-hours for the RL stages", "No hyperparameters for SFT/RL stages beyond the GRPO objective", "Stored source is an ar5iv conversion truncated at the author list — body sections complete (verified)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "deepseek-r1.json" }, { "slug": { "value": "deepseek-v3-2", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2512.02556", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/deepseek-v3-2", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "DeepSeek-V3.2", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "DeepSeek-AI", "provenance": "stated", "ref": "Title page byline (DeepSeek-AI)" }, "release_date": { "value": "2025-12-02", "provenance": "stated", "ref": "arXiv header ('arXiv:2512.02556v1 [cs.CL] 02 Dec 2025'); confirmed via export.arxiv.org API (published 2025-12-02)" }, "report_url": { "value": "https://arxiv.org/abs/2512.02556", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named for the model release; 'arXiv.org perpetual non-exclusive license' on the page covers the paper only" }, "open_weights": { "value": true, "provenance": "inferred", "note": "report frames DeepSeek-V3.2 as an open model ('Pushing the Frontier of Open Large Language Models', Title/Abstract) and links an open-source inference implementation at huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp (§2.1), but never states in so many words that the weights are released" } }, "scale": { "total_params": { "value": null, "provenance": "unknown", "note": "report prints no parameter counts; it continues from the DeepSeek-V3.1-Terminus base checkpoint (§2.1.1) without restating its size. Family context only: the atlas deepseek-v3 record reports 671B total for the V3 line, but that figure is not stated in this report" }, "active_params_per_token": { "value": null, "provenance": "unknown", "note": "not disclosed in this report" }, "layers": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: no dimensions disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§2.1", "note": "MLA: keys and values are jointly compressed into a latent vector (no separate KV heads); DSA is instantiated in the MQA mode of MLA where each latent vector is shared across all query heads" }, "head_dim": { "value": null, "provenance": "unknown", "note": "MLA latent dimension not restated in this report" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not stated" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "§3.1", "note": "report explicitly describes the model as MoE: 'Mixture-of-Experts (MoE) models improve computational efficiency by activating only a subset of expert modules during inference' (Keep Routing). Exact expert counts are not restated" }, "moe": { "expert_count": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "experts_per_token": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "not restated in this report" }, "routing": { "value": null, "provenance": "unknown", "note": "report discusses routing-path consistency between inference and training frameworks (Keep Routing, §3.1) but does not describe the router function" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "not described" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "not described" } }, "attention_variant": { "value": "MLA", "provenance": "stated", "ref": "§2.1", "note": "DSA is instantiated under MLA (cites DeepSeek-V2); for kernel efficiency the MQA mode of MLA is used, where each latent vector (key-value entry) is shared across all query heads of the query token" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "DSA replaces vanilla attention throughout the model ('the only architectural modification of DeepSeek-V3.2 is the introduction of DSA', §2.1); no per-layer variation described" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: architecture described as the DeepSeek-V3.1-Terminus block with DSA attention; no attention over prior layer outputs mentioned" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "report silent on positional encoding; unchanged from DeepSeek-V3.1-Terminus per §2.1 but not described here" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "continued pre-training runs at 128K sequences and the model serves 128K (§2.1.1, §4.1); the 128K extension itself predates this report (DeepSeek-V3.1-Terminus base 'whose context length has been extended to 128K') and is not described here" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.1.1", "note": "no rescaling factor applies; no extension method described in this report" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§4.1", "note": "'DeepSeek-V3.2 supports a maximum context length of only 128K'; evaluation sets the context window to 128K tokens" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "not described in this report" }, "placement": { "value": null, "provenance": "unknown", "note": "not described in this report" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "MLA attention internals beyond DSA are not described in this report" } }, "activation": { "value": null, "provenance": "unknown", "note": "not described in this report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: the attention mechanism is fully described (MLA + DSA top-k selection, §2.1) with no designed sink tokens" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described" }, "other": { "value": null, "provenance": "unknown", "note": "no block-level stability mechanisms described in this report; the RL-side stabilizers (unbiased KL estimate, off-policy sequence masking, keep routing, keep sampling mask, §3.1) are training-side measures" } } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§2.1.1", "note": "continued pre-training runs steps of 128K-token sequences (16 per step warm-up, 480 per step sparse stage); base checkpoint context already extended to 128K" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§4.1", "note": "'DeepSeek-V3.2 supports a maximum context length of only 128K'" }, "extension_method": { "value": "none", "provenance": "inferred", "note": "training at 128K equals deployed 128K; the 128K extension was performed for DeepSeek-V3.1-Terminus and is not described in this report" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer not described in this report" }, "notes": { "value": null, "provenance": "unknown", "note": "not described" } }, "training": { "tokens": { "value": 945.8, "provenance": "derived", "note": "sum of DSA continued pre-training tokens: 2.1B (dense warm-up: 1000 steps × 16 sequences × 128K) + 943.7B (sparse stage: 15000 steps × 480 sequences × 128K) = 945.8B (§2.1.1). The original pre-training token count is not restated in this report" }, "data_composition": { "value": "continued pre-training data distribution 'totally aligned with the 128K long context extension data used for DeepSeek-V3.1-Terminus' (§2.1.1); post-training data: specialist-distilled domain data (mathematics, programming, general logical reasoning, general agentic tasks, agentic coding, agentic search, plus writing and general QA, in thinking and non-thinking modes) and agentic RL data (real tools: web search API, coding tools, Jupyter Notebooks; synthesized: 1,827 task-oriented environments with 4,417 general-agent tasks, plus 24,667 code-agent, 50,275 search-agent, 5,908 code-interpreter tasks)", "provenance": "stated", "ref": "§2.1.1, §3, Table 1" }, "curriculum": { "value": "two-stage DSA continued pre-training: (1) dense warm-up freezing all parameters except the lightning indexer, trained with a KL objective aligned to the summed head-wise attention distribution (LR 1e-3, 1000 steps); (2) sparse training with fine-grained top-k selection, optimizing all parameters (LR 7.3e-6, 15000 steps, k=2048); then post-training: specialist distillation → mixed GRPO RL (reasoning + agent + alignment in one stage) → continued RL to final checkpoints", "provenance": "stated", "ref": "§2.1.1, §3" }, "optimizer": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "lr_schedule": { "value": "continued pre-training learning rates: 1e-3 (indexer warm-up stage), 7.3e-6 (sparse training stage)", "provenance": "stated", "ref": "§2.1.1", "note": "only the DSA continued pre-training LRs are given; no schedule for the original pre-training" }, "batch_schedule": { "value": "16 sequences of 128K tokens per step (dense warm-up); 480 sequences of 128K tokens per step (sparse stage)", "provenance": "stated", "ref": "§2.1.1" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not stated; the lightning indexer 'can be implemented in FP8' (§2.1) is a kernel-efficiency note" }, "parallelism": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed; H800 GPUs appear only in inference cost benchmarking ('the actual service deployed on H800 GPUs', §2.3)" }, "compute": { "value": null, "provenance": "unknown", "note": "no absolute FLOPs or GPU-hours; the report states the post-training computational budget 'exceeding 10% of the pre-training cost' (§1, §4.1)" } }, "post_training": { "sft": { "value": true, "provenance": "inferred", "note": "the report references 'the SFT checkpoint of DeepSeek-V3.2 (denoted DeepSeek-V3.2-SFT)' in the agentic-RL ablation (§4.3), implying an SFT stage in the pipeline; the pipeline itself is described as specialist distillation + mixed RL (§3)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§3", "note": "'we still adopt Group Relative Policy Optimization (GRPO) (deepseekmath; deepseekr1) as the RL training algorithm'" }, "reasoning_training": { "value": "mixed RL merging reasoning, agent, and human-alignment training into one GRPO stage; rule-based outcome reward, length penalty and language consistency reward for reasoning/agent tasks, generative reward model with per-prompt rubrics for general tasks; RL compute >10% of pre-training cost; specialist distillation producing thinking (long-CoT) and non-thinking data; RL stabilizers: unbiased (importance-sampling corrected) KL estimate, off-policy sequence masking, keep routing, keep sampling mask; thinking-in-tool-use via cold-start (DeepSeek-V3 methodology) and thinking context management (reasoning retained across tool messages, discarded only on new user messages, tool-call history preserved); DeepSeek-V3.2-Speciale variant trained exclusively on reasoning data with reduced length penalty plus DeepSeekMath-V2 dataset/rewards for mathematical proofs", "provenance": "stated", "ref": "§3, §3.1, §3.2" }, "distillation": { "value": "specialist distillation: per-domain specialist models (fine-tuned from the same pre-trained V3.2 base with large-scale RL) generate domain-specific data for the final checkpoint; 'models trained on the distilled data achieve performance levels only marginally below those of domain-specific specialists', with the gap eliminated by subsequent RL", "provenance": "stated", "ref": "§3" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes only text and function-call/tool interaction; no non-text modality appears anywhere (all evaluations are text-based)" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§1", "note": "text-only; no non-text modality" } }, "efficiency": { "kv_cache": { "value": "MLA: low-rank latent KV cache; DSA retrieves only the top-k=2048 key-value entries per query token, so attention computation touches a sparse subset of the latent cache; MQA-mode MLA shares each latent vector across all query heads for kernel efficiency", "provenance": "derived", "note": "consequence of attention_variant MLA + DSA top-k selection (§2.1); core attention complexity reduced from O(L²) to O(Lk), k=2048 (§2.3)" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "no shipped quantization format described; the lightning indexer 'can be implemented in FP8' (§2.1) is a computational-efficiency note, not a weight format" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding or multi-token prediction mentioned in this report (MTP exists in the V3 family but is not restated here)" }, "serving": { "value": "masked MHA mode simulating DSA for short-sequence prefilling (higher efficiency under short contexts), plus 'our optimized implementation' of DSA achieving significant end-to-end speedup in long-context scenarios; token-cost benchmarks from the deployed service on H800 clusters at $2 per GPU hour", "provenance": "stated", "ref": "§2.3" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro (EM)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 85.0, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA Diamond (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 82.4, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HLE (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 25.1, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "LiveCodeBench (Pass@1-COT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 83.3, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Codeforces (Rating)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 2386, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AIME 2025 (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 93.1, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HMMT Feb 2025 (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 92.5, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HMMT Nov 2025 (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 90.2, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "IMOAnswerBench (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 78.3, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Terminal Bench 2.0 (Acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 46.4, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking; Claude Code framework)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking; Claude Code framework)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SWE Verified (Resolved)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.1, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SWE Multilingual (Resolved)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 70.2, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BrowseComp (Pass@1, with/without context management)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "51.4/67.6*", "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking; * = with context management technique)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BrowseCompZh (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 65.0, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HLE (Pass@1, search agent)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 40.8, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "τ²-Bench (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 80.3, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking; average of categories: Airline 63.8, Retail 81.1, Telecom 96.2)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MCP-Universe (Success Rate)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 45.9, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MCP-Mark (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 38.0, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Tool-Decathlon (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 35.2, "provenance": "stated", "ref": "Table 2 (DeepSeek-V3.2 Thinking)" }, "ref": { "value": "Table 2 (DeepSeek-V3.2 Thinking)", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AIME 2025 (Pass@1, V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 96.0, "provenance": "stated", "ref": "Table 3 (DeepSeek-V3.2-Speciale, 23k output tokens)" }, "ref": { "value": "Table 3 (DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA Diamond (Pass@1, V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 85.7, "provenance": "stated", "ref": "Table 3 (DeepSeek-V3.2-Speciale)" }, "ref": { "value": "Table 3 (DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (Pass@1, V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 30.6, "provenance": "stated", "ref": "Table 3 (DeepSeek-V3.2-Speciale)" }, "ref": { "value": "Table 3 (DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Codeforces (Rating, V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 2701, "provenance": "stated", "ref": "Table 3 (DeepSeek-V3.2-Speciale)" }, "ref": { "value": "Table 3 (DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "IMO 2025 (points, V3.2-Speciale)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "35/42 (Gold)", "provenance": "stated", "ref": "Table 4 (DeepSeek-V3.2-Speciale)" }, "ref": { "value": "Table 4 (DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "IOI 2025 (points, V3.2-Speciale)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "492/600 (Gold, 10th place)", "provenance": "stated", "ref": "Table 4 (DeepSeek-V3.2-Speciale)" }, "ref": { "value": "Table 4 (DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "Table 4" } } ] }, "contributions": { "claimed": [ { "text": { "value": "DeepSeek Sparse Attention (DSA): a lightning indexer (few heads, ReLU-scored, FP8-implementable) plus fine-grained top-k token selection, cutting the main attention complexity from O(L²) to O(Lk) with k=2048 selected key-value entries per query, instantiated under MLA in MQA mode", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "A stable, scalable RL protocol: GRPO augmented with an unbiased KL estimate, off-policy sequence masking, keep routing, and keep sampling mask, supporting a post-training computational budget exceeding 10% of the pre-training cost", "provenance": "stated", "ref": "§1, §3.1" }, "ref": { "value": "§1, §3.1", "provenance": "stated", "ref": "§1, §3.1" } }, { "text": { "value": "Large-scale agentic task synthesis pipeline: over 1,800 synthesized environments and 85,000 complex prompts (24,667 code-agent, 50,275 search-agent, 4,417 general-agent, 5,908 code-interpreter tasks) for scalable agentic post-training", "provenance": "stated", "ref": "§1, §3.2.3" }, "ref": { "value": "§1, §3.2.3", "provenance": "stated", "ref": "§1, §3.2.3" } }, { "text": { "value": "Thinking context management for tool-use: historical reasoning content discarded only when a new user message arrives, with tool-call history preserved, avoiding redundant re-reasoning per tool call", "provenance": "stated", "ref": "§3.2.1" }, "ref": { "value": "§3.2.1", "provenance": "stated", "ref": "§3.2.1" } }, { "text": { "value": "DeepSeek-V3.2-Speciale: a relaxed-length, high-compute variant reaching gold-medal performance in IMO 2025, CMO 2025, IOI 2025 and ICPC World Finals 2025 (ranked 2nd), 'without targeted training'", "provenance": "stated", "ref": "§4.2, Table 4" }, "ref": { "value": "§4.2, Table 4", "provenance": "stated", "ref": "§4.2, Table 4" } }, { "text": { "value": "Parity claim: the DSA base model matches DeepSeek-V3.1-Terminus on standard benchmarks and human preference (closely matched ChatbotArena Elo, +4 points on AA-LCR) while substantially improving long-sequence computational efficiency", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } } ], "assessment": { "value": "DSA is the first sparse-attention mechanism in the atlas built on MLA: an auxiliary lightning indexer computes cheap ReLU scores, a frozen dense warm-up (1000 steps) KL-aligns indexer softmax scores to the summed head-wise attention distribution, and top-k=2048 selection then sparsifies every layer's attention to O(Lk). It is a training-stabilized retrieval-style sparsification, not a new mixing family — contrast Kimi K3's KDA/MLA layerwise hybrid and the GLM-4.5 attention variants. The report's own parity evidence (base model matches V3.1-Terminus) is the key architectural claim and is internally consistent. The GRPO stability recipe (unbiased KL, off-policy masking, keep-routing, keep-sampling) is a substantive engineering contribution beyond deepseek-r1's report. Everything else — MLA, DeepSeekMoE, 128K context — is inherited unchanged from the V3 line (stated explicitly), so the atlas deepseek-v3 record remains the family's disclosure source for parameters. The >10%-of-pre-training RL compute and competition gold medals are internal claims, not independently verifiable.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, deepseek-r1, moonshot-kimi-k3, zai-glm-4-5) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "stated", "ref": "§3.1", "note": "report explicitly describes the model as MoE ('activating only a subset of expert modules during inference', §3.1); exact expert counts not restated" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "DSA (top-k sparse attention, k=2048 per query) replaces vanilla attention in all layers (§2.1); core complexity O(Lk)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no non-text modality anywhere in the report)" }, "openness": { "value": "open-weights", "provenance": "inferred", "note": "report frames the model as open ('Pushing the Frontier of Open Large Language Models') and releases an open-source inference implementation (§2.1); no explicit weights-release statement and no data release" }, "scale_class": { "value": null, "provenance": "unknown", "note": "parameter counts are not disclosed in this report, so scale class cannot be derived here (family context: the atlas deepseek-v3 record reports 671B, but that is not stated in this report)" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "framed as harmonizing 'high computational efficiency with superior reasoning and agent performance'; post-training built on specialist distillation and reasoning RL; the Speciale variant explicitly pushes extended thinking (§1, §3)" } }, "prose": { "design_overview": { "value": "DeepSeek-V3.2 is a continued-trained member of the DeepSeek-V3 line: it starts from the DeepSeek-V3.1-Terminus base checkpoint (context already extended to 128K, architecture otherwise unchanged) and its only architectural modification is DeepSeek Sparse Attention (DSA). DSA replaces vanilla attention with a lightning indexer — a small number of heads computing a weighted ReLU score between each query and every preceding token, implementable in FP8 — and a fine-grained top-k selection that retrieves only the k=2048 best-scoring key-value entries per query, cutting the main attention cost from O(L²) to O(Lk). DSA is instantiated under MLA in its MQA mode (each latent KV vector shared across all query heads) for kernel efficiency; short-sequence prefilling uses a masked MHA mode that simulates DSA. The indexer is initialized in a frozen dense warm-up (1000 steps, 2.1B tokens) by KL-aligning its softmax scores to the summed head-wise attention distribution, then trained jointly in a 943.7B-token sparse stage. Post-training keeps the V3.2-Exp pipeline: six domain specialists plus writing and QA (thinking and non-thinking modes), distilled into the final model, then a single mixed GRPO stage fusing reasoning, agent, and alignment rewards, stabilized by an unbiased KL estimate, off-policy sequence masking, keep-routing, and keep-sampling masks; RL compute exceeds 10% of pre-training cost. Agentic capability comes from thinking context management (reasoning retained across tool messages) and a synthesis pipeline of 1,827 environments / 85,000 prompts. The relaxed-length DeepSeek-V3.2-Speciale variant, adding DeepSeekMath-V2 techniques, reaches gold-medal level at IMO/CMO/IOI 2025 and ranks 2nd at ICPC WF 2025.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Multi-head Latent Attention (MLA) — DSA is instantiated under MLA, with the MQA mode used for kernel efficiency", "provenance": "stated", "ref": "§2.1 (cites DeepSeek-V2, deepseekV2)" }, "origin": { "value": "DeepSeek-V2", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Base architecture with 128K context extension — DeepSeek-V3.2 starts from the DeepSeek-V3.1-Terminus base checkpoint and shares its architecture except for DSA", "provenance": "stated", "ref": "§2.1.1" }, "origin": { "value": "DeepSeek-V3.1-Terminus", "provenance": "stated", "ref": "§2.1.1" } }, { "mechanism": { "value": "Cold-start phase unifying reasoning and tool-use within single trajectories", "provenance": "stated", "ref": "§1 (cites DeepSeek-V3, deepseekv3)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§1" } }, { "mechanism": { "value": "Group Relative Policy Optimization (GRPO) as the RL training algorithm", "provenance": "stated", "ref": "§3 (cites DeepSeekMath, DeepSeek-R1)" }, "origin": { "value": "DeepSeek-R1 / DeepSeekMath", "provenance": "stated", "ref": "§3" } }, { "mechanism": { "value": "DeepSeekMath-V2 dataset and reward method for mathematical proofs (used in DeepSeek-V3.2-Speciale)", "provenance": "stated", "ref": "§3" }, "origin": { "value": "DeepSeekMath-V2", "provenance": "stated", "ref": "§3" } } ], "influence_out": [ { "model_slug": { "value": "deepseek-v4", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Lightning-indexer sparse attention (DSA)", "provenance": "inferred", "note": "mechanism as stated in deepseek-v4 report's lineage_in" } }, { "model_slug": { "value": "zai-glm-5", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "DeepSeek Sparse Attention (DSA) — content-based top-k sparse attention via a lightning indexer", "provenance": "inferred", "note": "mechanism as stated in zai-glm-5 report's lineage_in" } } ], "notable_omissions": { "value": [ "No parameter counts, active-parameter count, layer count, or hidden dims restated — the report refers to the DeepSeek-V3.1-Terminus base without numbers", "No MoE expert counts, router description, or load-balancing details (MoE acknowledged only in passing, §3.1)", "No positional-encoding, normalization, activation, or head-dimension details (deferred to the V3.1 architecture)", "No tokenizer or vocabulary description", "No training hardware, optimizer, precision, or absolute compute (FLOPs/GPU-hours); only the relative '>10% of pre-training cost' RL budget", "No license named for the release", "No explicit statement that model weights are released (only the open-source inference implementation link, §2.1)", "Number of indexer heads (H^I) and indexer dimension (d^I) not given numerically", "Top-k=2048 given for the sparse training stage only; the deployed k for inference is not stated", "The masked MHA mode for short-context prefill is named but not specified in detail", "No statement on DeepSeek-V3.2-Speciale availability/weights" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "deepseek-v3-2.json" }, { "slug": { "value": "deepseek-v3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2412.19437", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline (meta.json)" }, "stored_path": { "value": "data/sources/deepseek-v3", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "Title / Abstract" }, "variants": [], "org": { "value": "DeepSeek-AI", "provenance": "stated", "ref": "Title page author line" }, "release_date": { "value": "2024-12-27", "provenance": "inferred", "note": "arXiv v1 submission date (ID from report URL; arXiv API metadata) — report text itself is undated" }, "report_url": { "value": "https://arxiv.org/abs/2412.19437", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "report links checkpoints at github.com/deepseek-ai/DeepSeek-V3 (Abstract) but names no license" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('The model checkpoints are available at https://github.com/deepseek-ai/DeepSeek-V3')" } }, "scale": { "total_params": { "value": 671, "provenance": "stated", "ref": "Abstract / §4.2 / Table 3" }, "active_params_per_token": { "value": 37, "provenance": "stated", "ref": "Abstract / §4.2" }, "layers": { "value": 61, "provenance": "stated", "ref": "§4.2" }, "hidden_dim": { "value": 7168, "provenance": "stated", "ref": "§4.2" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "ref": "§4.2", "note": "intermediate hidden dimension of each MoE expert; no dense FFN width is disclosed (first three layers are dense)" }, "ffn_ratio": { "value": 0.3, "provenance": "derived", "note": "per-expert FFN ratio = 2048 / 7168 = 0.2857, rounded to one decimal" }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "§4.2" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§2.1.1", "note": "MLA: keys and values are jointly compressed into a low-rank latent (dc=512); no separate KV heads exist" }, "head_dim": { "value": 512, "provenance": "stated", "ref": "§4.2", "note": "MLA: the KV compression latent dimension dc (per schema §3, head_dim for MLA is the latent dimension)" }, "vocab_size": { "value": 131072, "provenance": "derived", "note": "report prints 'extended vocabulary of 128K tokens' (§4.1); no exact integer given; stored as 128 x 1024 = 131072" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "report never states input/output embedding tying; it only says the MTP modules share the embedding layer and the output head with the main model (§2.2)" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "§4.2", "note": "'We substitute all FFNs except for the first three layers with MoE layers'; 58 of 61 layers are MoE (value per data/schema.json enum)" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "§4.2" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§4.2", "note": "8 routed experts activated per token; during decoding the shared expert is additionally always selected (9 total), §3.4.2" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "§4.2" }, "routing": { "value": "sigmoid affinity scores (u_t^T e_i), top-8 selection, gating values normalized among selected experts", "provenance": "stated", "ref": "§2.1.2 (Eqs. 12-15)" }, "load_balancing": { "value": "auxiliary-loss-free bias-based balancing (per-expert bias b_i, bias update speed γ=0.001) plus a complementary sequence-wise auxiliary loss (α=0.0001); no token dropping", "provenance": "stated", "ref": "§2.1.2 / §4.2" }, "expert_granularity": { "value": "fine-grained", "provenance": "stated", "ref": "§2.1.2", "note": "'DeepSeekMoE uses finer-grained experts and isolates some experts as shared ones'" } }, "attention_variant": { "value": "MLA", "provenance": "stated", "ref": "§2.1.1" }, "attention_layer_pattern": { "value": "uniform", "provenance": "stated", "ref": "§2.1.1", "note": "MLA is used in all layers; no hybrid or windowed attention pattern is disclosed" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1.1 (Eq. 3)" }, "rope_base": { "value": null, "provenance": "unknown" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§2.1.1 (Eqs. 2-4, 7-9)", "note": "RoPE is applied only to the decoupled key/query vectors (per-head rotary dim d_h^R = 64); the compressed latent dims carry no RoPE" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "ref": "§4.3" }, "factor": { "value": 40, "provenance": "stated", "ref": "§4.3", "note": "YaRN scale s=40, with α=1, β=32, sqrt(t) = 0.1*ln(s) + 1; applied only to the decoupled shared key" }, "target_length": { "value": 131072, "provenance": "derived", "note": "report prints '128K' (§4.3); stored as 128 x 1024 = 131072" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.2 (Eq. 21) / §3.2.3 / §4.2" }, "placement": { "value": null, "provenance": "unknown", "note": "report does not state pre/post residual-branch placement; §2 says minor unmentioned details follow DeepSeek-V2, whose placement is not documented in this report" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§4.2", "note": "'additional RMSNorm layers after the compressed latent vectors' normalise the query and KV latents before up-projection (as in DeepSeek-V2)" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§3.3.3 ('Inputs of the SwiGLU operator in MoE')" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "report never mentions designed sink tokens; emergent sink behaviour would belong in prose" }, "softcapping": { "value": null, "provenance": "unknown", "note": "no logit softcapping disclosed" }, "other": { "value": [ "gradient clipping norm 1.0", "FP8 training keeps embedding, output head, MoE gating, normalization and attention operators in BF16/FP32, with master weights and gradients in FP32", "EMA of model parameters kept on CPU for early loss-decay estimates" ], "provenance": "stated", "ref": "§4.2 / §3.3.1 / §3.2.3" } }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: report describes its block in detail and never mentions attention over prior layer outputs; standard sequential residual connections are the default mechanism." } }, "context": { "trained_length": { "value": 4096, "provenance": "stated", "ref": "§4.2", "note": "maximum sequence length during pre-training is 4K" }, "deployed_length": { "value": 131072, "provenance": "derived", "note": "report prints '128K' (§4.3, two-stage extension 4K->32K->128K); stored as 128 x 1024 = 131072" }, "extension_method": { "value": "YaRN, two-stage: 4K to 32K then 32K to 128K, 1000 steps per phase (scale s=40)", "provenance": "stated", "ref": "§4.3" } }, "tokenizer": { "algorithm": { "value": "Byte-level BPE", "provenance": "stated", "ref": "§4.1" }, "notes": { "value": "Extended vocabulary of 128K tokens; pretokenizer and tokenizer training data modified for multilingual compression efficiency; pretokenizer adds combined punctuation-and-linebreak tokens, mitigated by randomly splitting a proportion of such tokens during training; Fill-in-the-Middle applied at rate 0.1 via the Prefix-Suffix-Middle (PSM) framework", "provenance": "stated", "ref": "§4.1" } }, "training": { "tokens": { "value": 14.8, "provenance": "stated", "ref": "Abstract / §4.2" }, "data_composition": { "value": "14.8T tokens; versus DeepSeek-V2 the corpus raises the ratio of mathematical and programming samples and expands multilingual coverage beyond English and Chinese; processing pipeline refined to minimise redundancy while preserving diversity; document packing without cross-sample attention masking; FIM at rate 0.1 (PSM). Exact mixture percentages are not disclosed", "provenance": "stated", "ref": "§4.1" }, "curriculum": { "value": "Single pre-training run at 4K sequence length with a phased LR schedule (linear warmup, constant, cosine decay, final constant); then two-stage context extension (32K, 128K); then post-training (SFT, RL)", "provenance": "stated", "ref": "§4.2 / §4.3" }, "optimizer": { "value": "AdamW (beta1=0.9, beta2=0.95, weight_decay=0.1, gradient clipping norm 1.0)", "provenance": "stated", "ref": "§4.2" }, "lr_schedule": { "value": "linear warmup 0 to 2.2e-4 over the first 2K steps; constant 2.2e-4 until 10T tokens; cosine decay to 2.2e-5 over 4.3T tokens; final 500B tokens: constant 2.2e-5 for the first 333B then 7.3e-6 for the remaining 167B", "provenance": "stated", "ref": "§4.2" }, "batch_schedule": { "value": "batch size gradually increased from 3072 to 15360 over the first 469B tokens, then held at 15360", "provenance": "stated", "ref": "§4.2" }, "precision": { "value": "FP8 mixed precision: GEMMs in FP8 (E4M3 format on all tensors); embedding, output head, MoE gating, normalization and attention kept in BF16/FP32; master weights and gradients in FP32; optimizer first/second moments in BF16; activations cached FP8 (custom E5M6 for inputs of the Linear after attention); fine-grained quantization (1x128 activation tiles, 128x128 weight blocks) with online scaling; promotion to CUDA cores every Nc=128 elements for FP32 accumulation", "provenance": "stated", "ref": "§3.3" }, "parallelism": { "value": "16-way pipeline parallelism (DualPipe, bidirectional), 64-way expert parallelism spanning 8 nodes, ZeRO-1 data parallelism; no tensor parallelism in training", "provenance": "stated", "ref": "§3.2" }, "hardware": { "value": "2048 NVIDIA H800 GPUs; nodes of 8 GPUs with NVLink/NVSwitch (NVLink 160 GB/s); InfiniBand (50 GB/s) across nodes", "provenance": "stated", "ref": "§3.1 / §3.2.2" }, "compute": { "value": "2.788M H800 GPU hours total (2664K pre-training + 119K context extension + 5K post-training); ~$5.576M at $2 per GPU hour; 180K GPU hours per trillion tokens", "provenance": "stated", "ref": "Table 1 / §1" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§5.1", "note": "1.5M curated instruction instances across multiple domains; two epochs with cosine LR 5e-6 to 1e-6; packed sequences with sample masking" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§5.2.2" }, "reasoning_training": { "value": "Long-CoT reasoning data distilled from the DeepSeek-R1 series (reflection/verification patterns); SFT data curated from RL-trained expert model generators via rejection sampling; GRPO RL with rule-based and model-based reward models; self-rewarding via constitutional-AI-style voting", "provenance": "stated", "ref": "§5.1 / §5.2 / §5.4.2" }, "distillation": { "value": "DeepSeek-R1 series: reasoning capability distilled into DeepSeek-V3 via R1-generated SFT data and expert-model rejection sampling", "provenance": "stated", "ref": "§5.1 / §5.4.1" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes only text input/output; every disclosed evaluation task is text-based (§4.4, §5.3); no vision/audio modality described" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§4.4.1", "note": "text-only model; no non-text modality attaches" } }, "efficiency": { "kv_cache": { "value": "MLA low-rank latent KV cache: only the compressed KV latent (dc=512) and the decoupled RoPE key (64 dims per head) are cached per token, sharply reducing KV cache versus MHA", "provenance": "stated", "ref": "§2.1.1" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "§3.4", "note": "no inference quantisation formats disclosed; FP8 is used for training (§3.3)" }, "speculative_dedup": { "value": "MTP modules repurposable for speculative decoding: second-token acceptance rate 85-90%, ~1.8x decoding speed (tokens per second)", "provenance": "stated", "ref": "§5.4.3 / §2.2" }, "serving": { "value": "Separate prefill/decode deployment: prefill TP4+SP with DP8 and EP32 (4 nodes/32 GPUs, 32 redundant experts); decode TP4+SP with DP80 and EP320 (40 nodes/320 GPUs, one expert per GPU, shared expert treated as always-selected routed expert); redundant high-load expert duplication with periodic rebalancing; dynamic redundancy explored; IBGDA; dual micro-batch overlap of attention and MoE dispatch/combine", "provenance": "stated", "ref": "§3.4" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (EM)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 88.5, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMLU-Redux (EM)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 89.1, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMLU-Pro (EM)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 75.9, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GPQA-Diamond (Pass@1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 59.1, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SimpleQA (Correct)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 24.9, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "C-SimpleQA (Correct)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 64.8, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "DROP (3-shot F1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 91.6, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "LongBench v2 (Acc.)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 48.7, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "HumanEval-Mul (Pass@1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 82.6, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "LiveCodeBench (Pass@1-COT)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 40.5, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Codeforces (Percentile)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 51.6, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SWE Verified (Resolved)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 42.0, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "AIME 2024 (Pass@1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 39.2, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MATH-500 (EM)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 90.2, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "CNMO 2024 (Pass@1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 43.2, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Pioneers an auxiliary-loss-free load balancing strategy for DeepSeekMoE: per-expert bias terms adjusted each step (bias update speed gamma=0.001) steer top-k routing without an auxiliary loss, avoiding the performance penalty of auxiliary-loss-based balancing; a small sequence-wise auxiliary loss (alpha=0.0001) only prevents extreme within-sequence imbalance, and no tokens are dropped", "provenance": "stated", "ref": "§2.1.2 / §4.2" }, "ref": { "value": "§2.1.2 / §4.2", "provenance": "stated", "ref": "§2.1.2 / §4.2" } }, { "text": { "value": "Investigates a Multi-Token Prediction (MTP) training objective using sequential, causality-preserving modules with a shared embedding layer and output head; it improves evaluation performance, and the MTP modules can be repurposed for speculative decoding (second-token acceptance 85-90%, ~1.8x TPS)", "provenance": "stated", "ref": "§2.2 / §5.4.3" }, "ref": { "value": "§2.2 / §5.4.3", "provenance": "stated", "ref": "§2.2 / §5.4.3" } }, { "text": { "value": "Introduces an FP8 mixed-precision training framework and, for the first time, validates FP8 training on an extremely large-scale model (fine-grained 1x128/128x128 quantization with online scaling, E4M3 everywhere, CUDA-core promotion for FP32 accumulation; relative loss error below 0.25% vs BF16)", "provenance": "stated", "ref": "§3.3 / Appendix B" }, "ref": { "value": "§3.3 / Appendix B", "provenance": "stated", "ref": "§3.3 / Appendix B" } }, { "text": { "value": "Designs DualPipe, a bidirectional pipeline-parallel algorithm with fewer bubbles and full computation-communication overlap, plus custom cross-node all-to-all kernels over IB and NVLink, achieving near-zero communication overhead for fine-grained cross-node experts and enabling training without tensor parallelism", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } }, { "text": { "value": "Completes pre-training of DeepSeek-V3 on 14.8T tokens at an economical cost of 2.664M H800 GPU hours (2.788M including context extension and post-training, ~$5.576M total), producing the strongest open-source base model at the time of writing, with no irrecoverable loss spikes or rollbacks", "provenance": "stated", "ref": "§1 / Table 1" }, "ref": { "value": "§1 / Table 1", "provenance": "stated", "ref": "§1 / Table 1" } }, { "text": { "value": "Introduces a post-training methodology that distills reasoning capability from a long-CoT DeepSeek-R1 series model into a standard LLM, incorporating R1's verification and reflection patterns while maintaining control over output style and length", "provenance": "stated", "ref": "§5 / §5.4.1" }, "ref": { "value": "§5 / §5.4.1", "provenance": "stated", "ref": "§5 / §5.4.1" } } ], "assessment": { "value": "The core block is inherited from DeepSeek-V2 — MLA and DeepSeekMoE are adopted wholesale, so the genuinely new mechanisms are the auxiliary-loss-free bias-based load balancing (previously published by DeepSeek's own group, Wang et al. 2024a, per the report) and the sequential MTP objective (a causality-preserving variant of Gloeckle et al.'s parallel multi-token prediction). Against other atlas entries, DeepSeek-V3 is distinctive for low-rank latent KV compression versus GQA (Llama 3.1) or plain top-k MoE (Mixtral), for fine-grained 256-expert routing with a shared expert, and for a training scheme that avoids tensor parallelism; the FP8 framework and DualPipe are engineering contributions whose novelty lies mainly in the 671B-scale validation. The report's 'pioneers' and best-in-class open-source claims rest on its own internal evaluation harness, and the closed-source comparisons (GPT-4o, Claude-3.5-Sonnet) are self-reported, so headline numbers should be read with the same caution as any vendor report.", "provenance": "inferred", "note": "authored by atlas analyst; comparison against other atlas records (Llama 3.1 GQA, Mixtral MoE) and report-internal citations" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "58 of 61 transformer layers use routed MoE FFNs (first three layers dense, §4.2); MoE share ~95% of layers, above the 80% majority rule (taxonomy.md Axis 1)" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "MLA in every layer with no windowed/sparse/SSM pattern disclosed; attention_layer_pattern is uniform (§2.1.1)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type: report describes no non-text modality" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "report states checkpoints are available (Abstract) and describes the data mixture without releasing it; no license named (taxonomy.md Axis 4)" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "671B total parameters >= 100B (taxonomy.md Axis 5)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "report frames DeepSeek-V3 as a general-purpose open-source LLM competitive with GPT-4o and Claude-3.5-Sonnet across knowledge, code, math, long-context and Chinese benchmarks (Abstract, §1, §5.3); R1 reasoning distillation is a post-training ingredient, not the lead framing" } }, "prose": { "design_overview": { "value": "DeepSeek-V3 is a 61-layer Transformer with a sparse mixture-of-experts block: the first three layers keep dense FFNs while the remaining 58 replace them with DeepSeekMoE layers of one shared expert and 256 fine-grained routed experts (per-expert width 2048), eight of which fire per token — 671B total, 37B active parameters. Attention is Multi-head Latent Attention (MLA) carried over from DeepSeek-V2: keys and values are jointly compressed into a 512-dimensional latent plus a 64-dimensional decoupled RoPE key per head, so the KV cache stores only the latent and the rotary key per token, and queries are low-rank compressed (1536-dim) to cut training activations. Load balancing is auxiliary-loss-free: per-expert biases steer sigmoid-gated top-8 routing and are nudged by a bias-update speed of 0.001, with a tiny sequence-wise auxiliary loss (alpha=0.0001) guarding against within-sequence imbalance; node-limited routing sends each token to at most four nodes, and no tokens are dropped in training or inference. A depth-1 Multi-Token Prediction module — shared embedding and output head with RMSNorm-gated fusion — adds a second-token loss and doubles as a speculative-decoding draft (85-90% acceptance, 1.8x tokens per second). Training on 2048 H800 GPUs combines FP8 mixed precision (E4M3, fine-grained tile/block quantization, CUDA-core accumulation), DualPipe 16-way pipeline parallelism, 64-way expert parallelism, and no tensor parallelism, over 14.8T tokens, then YaRN-based two-stage extension to 128K, then SFT and GRPO RL with reasoning distilled from DeepSeek-R1. The design's significance is cost: about 2.788M GPU hours (roughly $5.6M) for a frontier-competitive open-weight model.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "MLA (multi-head latent attention)", "provenance": "stated", "ref": "§2.1.1" }, "origin": { "value": "DeepSeek-V2", "provenance": "stated", "ref": "§1" } }, { "mechanism": { "value": "DeepSeekMoE (fine-grained routed experts plus isolated shared experts)", "provenance": "stated", "ref": "§2.1.2" }, "origin": { "value": "DeepSeek-V2 (Dai et al. 2024)", "provenance": "stated", "ref": "§1" } }, { "mechanism": { "value": "Auxiliary-loss-free bias-based load balancing", "provenance": "stated", "ref": "§2.1.2" }, "origin": { "value": "Wang et al. 2024a (DeepSeek auxiliary-loss-free MoE work)", "provenance": "stated", "ref": "§2.1.2" } }, { "mechanism": { "value": "Multi-token prediction (MTP) training objective", "provenance": "stated", "ref": "§2.2" }, "origin": { "value": "Gloeckle et al. 2024 (parallel MTP; V3 makes it sequential)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Causal-chain speculative draft (EAGLE-style)", "provenance": "stated", "ref": "§2.2" }, "origin": { "value": "EAGLE (Li et al. 2024b)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "YaRN context extension (applied to the decoupled MLA key)", "provenance": "stated", "ref": "§4.3" }, "origin": { "value": "Peng et al. 2023a (YaRN)", "provenance": "stated", "ref": "§4.3" } }, { "mechanism": { "value": "GRPO (group relative policy optimization)", "provenance": "stated", "ref": "§5.2.2" }, "origin": { "value": "DeepSeekMath (Shao et al. 2024)", "provenance": "stated", "ref": "§5.2.2" } } ], "influence_out": [ { "model_slug": { "value": "moonshot-kimi-k3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "MLA lineage: K3 retains MLA in its periodic Gated MLA layers (report cites DeepSeek-V2)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "model_slug": { "value": "moonshot-kimi-k3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Shared + fine-grained routed expert organization: Stable LatentMoE follows the DeepSeekMoE structure (report cites DeepSeek-V3)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "model_slug": { "value": "moonshot-kimi-k3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Auxiliary-loss-free bias-based load balancing: Quantile Balancing generalizes DeepSeek-V3's bias updates to a quantile rule (report cites DeepSeek-V3)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "model_slug": { "value": "zai-glm-4-5-air", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Sigmoid gating and loss-free (auxiliary-loss-free) balance routing for MoE layers", "provenance": "inferred", "note": "mechanism as stated in zai-glm-4-5-air report's lineage_in" } }, { "model_slug": { "value": "deepseek-v4", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "DeepSeekMoE (shared + routed experts) and MTP", "provenance": "inferred", "note": "mechanism as stated in deepseek-v4 report's lineage_in" } }, { "model_slug": { "value": "zai-glm-4-5", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Sigmoid gating for MoE layers", "provenance": "inferred", "note": "mechanism as stated in zai-glm-4-5 report's lineage_in" } }, { "model_slug": { "value": "nvidia-nemotron-3-nano-30b-a3b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Aux-loss-free bias-based load balancing (expert-bias update rate 10−3)", "provenance": "inferred", "note": "mechanism as stated in nvidia-nemotron-3-nano-30b-a3b report's lineage_in" } }, { "model_slug": { "value": "zyphra-zaya1-8b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Auxiliary-loss-free bias-based load balancing — PID-style update of the DeepSeek bias scheme", "provenance": "inferred", "note": "mechanism as stated in zyphra-zaya1-8b report's lineage_in" } }, { "model_slug": { "value": "poolside-laguna-xs-2-33b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Shared + routed experts with routed-expert modulation ('similar to DeepSeek-V3')", "provenance": "inferred", "note": "mechanism as stated in poolside-laguna-xs-2-33b report's lineage_in" } }, { "model_slug": { "value": "deepseek-v3-2", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Base architecture with 128K context extension — DeepSeek-V3.2 starts from the DeepSeek-V3.1-Terminus base checkpoint and shares its architecture except for DSA", "provenance": "inferred", "note": "mechanism as stated in deepseek-v3-2 report's lineage_in" } }, { "model_slug": { "value": "moonshot-kimi-k2", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Multi-head Latent Attention (MLA) with partial rotary encoding", "provenance": "inferred", "note": "mechanism as stated in moonshot-kimi-k2 report's lineage_in" } }, { "model_slug": { "value": "xiaomi-mimo-v2-flash-309b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "FP8 mixed-precision framework (BF16 attention output projections and embeddings/output head, FP32 router)", "provenance": "inferred", "note": "mechanism as stated in xiaomi-mimo-v2-flash-309b report's lineage_in" } }, { "model_slug": { "value": "minimax-m2", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Multi-Token Prediction (MTP) module following the DeepSeek-V3 design during pre-training", "provenance": "inferred", "note": "mechanism as stated in minimax-m2 report's lineage_in" } }, { "model_slug": { "value": "arcee-ai-trinity-large-400b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Fine-grained routed experts with always-active shared expert (DeepSeekMoE design)", "provenance": "inferred", "note": "mechanism as stated in arcee-ai-trinity-large-400b report's lineage_in" } }, { "model_slug": { "value": "zai-glm-5", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Multi-Token Prediction (MTP) with a single (here parameter-shared) MTP layer as speculative draft", "provenance": "inferred", "note": "mechanism as stated in zai-glm-5 report's lineage_in" } }, { "model_slug": { "value": "deepseek-r1", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "MoE base architecture (671B total / 37B activated)", "provenance": "inferred", "note": "mechanism as stated in deepseek-r1 report's lineage_in" } }, { "model_slug": { "value": "nvidia-nemotron-3-super-120b-a12b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Multi-Token Prediction (MTP) for quality and speculative decoding", "provenance": "inferred", "note": "mechanism as stated in nvidia-nemotron-3-super-120b-a12b report's lineage_in" } } ], "notable_omissions": { "value": [ "No exact training-data mixture (domain percentages) — only a qualitative description (raised math/code ratio, expanded multilingual coverage)", "No license named for the released checkpoints", "No RoPE base frequency disclosed", "No explicit statement of pre-/post-normalization placement or residual structure", "No exact vocabulary size integer (only '128K tokens')", "No architecture or FP8 ablations at the full 671B scale — ablations use ~16B and ~230B stand-ins (Tables 4-5, Appendix B)", "No detailed reward-model architecture (only 'trained from the DeepSeek-V3 SFT checkpoints')", "No absolute inference latency/throughput numbers — only relative claims (2x DeepSeek-V2 generation speed; 1.8x TPS via MTP)", "No evaluation harness or prompts released with the report (internal HAI-LLM framework)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "deepseek-v3.json" }, { "slug": { "value": "deepseek-v4", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2606.19348", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/deepseek-v4", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "DeepSeek-V4", "provenance": "stated", "ref": "Title (report: 'Towards Highly Efficient Million-Token Context Intelligence')" }, "variants": [ { "value": "V4-Flash (284B)", "provenance": "stated", "ref": "Abstract / config tables" }, { "value": "V4-Pro (1.6T)", "provenance": "stated", "ref": "Abstract / config tables" } ], "org": { "value": "DeepSeek-AI", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-04-26", "provenance": "inferred", "note": "Report text undated; arXiv v1 submission date per export.arxiv.org API (arXiv:2606.19348)" }, "report_url": { "value": "https://arxiv.org/abs/2606.19348", "provenance": "stated", "ref": "pipeline" }, "license": { "provenance": "unknown" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract (checkpoints released on Hugging Face)" } }, "scale": { "total_params": { "value": 1600, "provenance": "stated", "ref": "config tables (V4-Pro 1.6T)" }, "active_params_per_token": { "value": 49, "provenance": "stated", "ref": "config tables (V4-Pro 49B active)" }, "layers": { "value": 61, "provenance": "stated", "ref": "config tables" }, "hidden_dim": { "value": 7168, "provenance": "stated", "ref": "config tables" }, "ffn_inner_dim": { "value": 3072, "provenance": "stated", "note": "per-expert inner dimension (MoE), config tables", "ref": "config tables" }, "ffn_ratio": { "value": 0.43, "provenance": "derived", "note": "3072 / 7168 = 0.43 (per-expert ratio)" }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "config tables (128 Q heads, head_dim 512)" }, "attention_heads_kv": { "provenance": "unknown", "note": "Compressed-attention design with shared-KV MQA; no conventional KV head count disclosed" }, "head_dim": { "value": 512, "provenance": "stated", "ref": "config tables" }, "vocab_size": { "value": 128000, "provenance": "stated", "ref": "config tables (128K)" }, "embedding_tied": { "provenance": "unknown" }, "variants": [ { "name": { "value": "V4-Flash (284B)", "provenance": "stated", "ref": "config tables" }, "total_params": { "value": 284, "provenance": "stated", "ref": "config tables" }, "layers": { "value": 43, "provenance": "stated", "ref": "config tables" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "config tables" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "note": "per-expert inner dimension", "ref": "config tables" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "config tables" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract (1M context)" } }, { "name": { "value": "V4-Pro (1.6T)", "provenance": "stated", "ref": "config tables" }, "total_params": { "value": 1600, "provenance": "stated", "ref": "config tables" }, "layers": { "value": 61, "provenance": "stated", "ref": "config tables" }, "hidden_dim": { "value": 7168, "provenance": "stated", "ref": "config tables" }, "ffn_inner_dim": { "value": 3072, "provenance": "stated", "note": "per-expert inner dimension", "ref": "config tables" }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "config tables" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract (1M context)" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "MoE layers in all Transformer blocks (report states inheritance of DeepSeekMoE)" }, "moe": { "expert_count": { "value": 384, "provenance": "stated", "ref": "config tables (routed; plus 1 shared)" }, "experts_per_token": { "value": 6, "provenance": "stated", "ref": "config tables" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "config tables" }, "routing": { "value": "Sqrt(Softplus) affinity; hash routing in first 3 MoE layers", "provenance": "stated", "ref": "architecture section" }, "load_balancing": { "value": "aux-loss-free load balancing + sequence balance loss", "provenance": "stated", "ref": "architecture section" }, "expert_granularity": { "provenance": "unknown" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "note": "Compressed sparse attention (CSA) / heavily-compressed attention (HCA) / sliding-window mix — no full-softmax layer", "ref": "architecture section" }, "attention_layer_pattern": { "value": "First 2 layers pure sliding-window (window 128); remaining layers interleaved CSA (KV compressed every m=4 tokens, overlapping dual compressed streams, with DeepSeek Sparse Attention lightning-indexer top-k) and HCA (heavy compression m'=128, dense attention)", "provenance": "stated", "ref": "architecture section" }, "depth_mixing": { "value": "hyper-connections", "provenance": "stated", "ref": "architecture section (mHC: Manifold-Constrained Hyper-Connections; residual matrix constrained to doubly stochastic manifold via Sinkhorn-Knopp, n_hc=4)" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "architecture section (partial)" }, "rope_base": { "provenance": "unknown" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "architecture section (RoPE on last 64 dims, with -i RoPE on outputs)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "Staged sequence-length curriculum 4K→16K→64K→1M", "ref": "architecture section" }, "factor": { "provenance": "unknown" }, "target_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "architecture section" }, "placement": { "provenance": "unknown" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "architecture section (RWKV-style RMSNorm on queries and compressed KV entries, explicitly replacing QK-Clip)" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "architecture section (with clamping)" }, "stability": { "attention_sinks": { "value": true, "provenance": "stated", "ref": "architecture section (designed attention sinks)" }, "softcapping": { "provenance": "unknown" }, "other": { "value": [ "SwiGLU clamping", "Anticipatory Routing", "mHC doubly-stochastic residual constraint" ], "provenance": "stated", "ref": "architecture section" } } }, "context": { "trained_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract (final curriculum stage trains at 1M)" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract (million-token context)" }, "extension_method": { "value": "Staged sequence-length curriculum 4K→16K→64K→1M", "provenance": "stated", "ref": "architecture section" } }, "tokenizer": { "algorithm": { "provenance": "unknown" }, "notes": { "provenance": "unknown" } }, "training": { "tokens": { "value": 33, "provenance": "stated", "ref": "config tables (V4-Pro 33T; V4-Flash 32T)" }, "data_composition": { "provenance": "unknown" }, "curriculum": { "value": "Sequence-length staging 4K→16K→64K→1M", "provenance": "stated", "ref": "architecture section" }, "optimizer": { "value": "Muon with hybrid Newton-Schulz", "provenance": "stated", "ref": "training section" }, "lr_schedule": { "value": "Cosine-style decay 2.0e-4 → 2.0e-5 (V4-Pro)", "provenance": "stated", "ref": "training section" }, "batch_schedule": { "value": "Batch 94.4M tokens (V4-Pro) / 75.5M (V4-Flash)", "provenance": "stated", "ref": "training section" }, "precision": { "provenance": "unknown" }, "parallelism": { "provenance": "unknown" }, "hardware": { "provenance": "unknown" }, "compute": { "provenance": "unknown" } }, "post_training": { "sft": { "provenance": "unknown" }, "preference_optimization": { "value": "other", "provenance": "inferred", "note": "OPD (on-policy distillation) and 'Quick Instruction' stages described; no RLHF/DPO/GRPO named" }, "reasoning_training": { "value": "none disclosed", "provenance": "inferred", "note": "Report emphasises long-context efficiency; no reasoning RL disclosed" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "OPD is a post-training distillation stage; teacher identity not disclosed" } }, "modality": { "type": { "value": "text-only", "provenance": "stated", "ref": "§6 (multimodal capabilities 'also working on')" }, "attachment": { "provenance": "n/a", "ref": "§6" } }, "efficiency": { "kv_cache": { "value": "CSA compresses KV every m=4 tokens (overlapping dual streams) + HCA m'=128 + shared-KV MQA; FP4 (MXFP4) QAT on expert weights and indexer QK path", "provenance": "derived", "note": "from attention pattern and quantization sections" }, "quantization": { "value": "FP4 (MXFP4) quantization-aware training on expert weights and indexer QK path", "provenance": "stated", "ref": "quantization section" }, "speculative_dedup": { "value": "MTP (depth 1)", "provenance": "stated", "ref": "architecture section" }, "serving": { "provenance": "unknown" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 90.1, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 73.5, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "SuperGPQA", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 53.9, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "SimpleQA-verified", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 55.2, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "HumanEval", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 76.8, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "MATH", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 64.5, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "LongBench-V2", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 51.5, "provenance": "stated", "ref": "eval tables (V4-Pro-Base)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 90.1, "provenance": "stated", "ref": "eval tables (V4-Pro-Max)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "SWE-Verified", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 80.6, "provenance": "stated", "ref": "eval tables (V4-Pro-Max)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "MRCR-1M", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 83.5, "provenance": "stated", "ref": "eval tables (V4-Pro-Max)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "CorpusQA-1M", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 62.0, "provenance": "stated", "ref": "eval tables (V4-Pro-Max)" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } } ] }, "contributions": { "claimed": [ { "text": { "value": "CSA (compressed sparse attention) and HCA (heavy compression) enabling efficient million-token context", "provenance": "stated", "ref": "Abstract / architecture section" }, "ref": { "value": "Abstract / architecture section", "provenance": "stated", "ref": "Abstract / architecture section" } }, { "text": { "value": "mHC: Manifold-Constrained Hyper-Connections residual mechanism (doubly-stochastic connection matrix)", "provenance": "stated", "ref": "architecture section" }, "ref": { "value": "architecture section", "provenance": "stated", "ref": "architecture section" } }, { "text": { "value": "FP4 (MXFP4) quantization-aware training as a first-class training target", "provenance": "stated", "ref": "quantization section" }, "ref": { "value": "quantization section", "provenance": "stated", "ref": "quantization section" } }, { "text": { "value": "Full replacement of dense attention with compressed/sparse attention — no full-softmax layer remains", "provenance": "stated", "ref": "architecture section" }, "ref": { "value": "architecture section", "provenance": "stated", "ref": "architecture section" } } ], "assessment": { "value": "The most aggressive efficiency-first evolution of the DeepSeek line in the atlas: MLA (V3) and DSA (V3.2) are superseded by a compressed sparse-attention stack, and the residual stream gains the first Hyper-Connections mechanism in the atlas (triggered the v0.4.0 depth_mixing schema extension). Novelty is high relative to atlas peers; the architecture section is dense and specific.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "from block: MoE in all Transformer blocks" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "CSA/HCA/SWA everywhere; no full-softmax layer (stated)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "1.6T reference variant" }, "design_intent": { "value": "long-context", "provenance": "inferred", "note": "title and framing: 'Towards Highly Efficient Million-Token Context Intelligence'" } }, "prose": { "design_overview": { "value": "DeepSeek V4 is the line's efficiency overhaul: it removes dense attention entirely. The first two layers are pure sliding-window (window 128); the rest interleave CSA — which compresses KV every 4 tokens into two overlapping streams and then applies the lightning-indexer sparse attention from V3.2 — with HCA, a heavy-compression (m'=128) dense attention. MLA and QK-Clip are gone, replaced by RWKV-style RMSNorm on queries and compressed KV entries (qk_norm true). The residual stream uses mHC, Hyper-Connections whose connection matrix is constrained to the doubly-stochastic manifold via Sinkhorn-Knopp — a new depth-mixing mechanism for the atlas (v0.4.0). MoE keeps DeepSeekMoE's shared-expert structure (1 shared + 384 routed, top-6, Sqrt(Softplus) affinity) with FP4 QAT training. V4-Pro is 1.6T/49B active over 61 layers; V4-Flash is 284B/13B over 43. NOTE: the gallery's HF-hosted DeepSeek_V4.pdf URL is dead (404); the report lives on arXiv 2606.19348.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "DeepSeekMoE (shared + routed experts) and MTP", "provenance": "stated", "ref": "architecture section (report: 'inherit DeepSeekMoE, MTP')" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "architecture section" } }, { "mechanism": { "value": "Lightning-indexer sparse attention (DSA)", "provenance": "stated", "ref": "architecture section" }, "origin": { "value": "DeepSeek-V3.2", "provenance": "stated", "ref": "architecture section" } } ], "influence_out": [], "notable_omissions": { "value": [ "The gallery-listed HF PDF URL (DeepSeek_V4.pdf) is a 404 — record uses the arXiv version", "No tokenizer algorithm disclosed", "No training precision/parallelism/hardware/compute disclosed", "Data mixture not disclosed", "SFT and RL details of post-training thin (OPD unnamed in detail)", "ConvKV/CA (Lightning Indexer) attention detail deferred to figures" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "deepseek-v4.json" }, { "slug": { "value": "google-gemma-3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2503.19786", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/google-gemma-3/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Gemma 3", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "1B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "4B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "12B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "27B", "provenance": "stated", "ref": "Abstract, Table 1" } ], "org": { "value": "Google DeepMind (Gemma Team)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2025-03-25", "provenance": "stated", "ref": "Header: 'arXiv:2503.19786v1 [cs.CL] 25 Mar 2025'" }, "report_url": { "value": "https://arxiv.org/abs/2503.19786", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "arXiv page prints 'License: CC BY 4.0' for the paper itself; the report never names a license for the model weights" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We release all our models to the community.'" } }, "scale": { "total_params": { "value": 27.0, "provenance": "stated", "ref": "Abstract, Table 1", "note": "Report labels the flagship 27B; Table 1 components sum to 27,016M (1,416M embedding + 25,600M non-embedding)" }, "active_params_per_token": { "value": 27.0, "provenance": "derived", "note": "Dense model: active = total (27B); Table 5 lists Gemma-3-27B-IT as 'Dense' with #params/#activated 27B" }, "layers": { "value": null, "provenance": "unknown", "note": "Report discloses no layer count for any variant (no config table; Table 1 gives parameter counts only)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed anywhere in the report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed; the report never names the FFN activation either" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "Cannot derive without ffn_inner_dim and hidden_dim" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Query head count not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "GQA is stated (§2) but the number of KV heads is not disclosed" }, "head_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed; cannot derive without hidden_dim and head counts" }, "vocab_size": { "value": 262000, "provenance": "stated", "ref": "§2.2 Tokenizer", "note": "Report prints 'The resulting vocabulary has 262k entries' (§2.2); Table 1 caption prints 'Our vocabulary has 256k entries' (inconsistent); the exact count is never printed" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "variants": [ { "name": { "value": "1B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 1.0, "provenance": "derived", "note": "302M embedding + 698M non-embedding = 1,000M (Table 1); no vision encoder (0)" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": 32768, "provenance": "stated", "ref": "§2 Long context ('with the exception of the 1B model that has 32K')" } }, { "name": { "value": "4B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 4.301, "provenance": "derived", "note": "417M vision + 675M embedding + 3,209M non-embedding = 4,301M (Table 1)" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§2 Long context, §5.3" } }, { "name": { "value": "12B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 12.188, "provenance": "derived", "note": "417M vision + 1,012M embedding + 10,759M non-embedding = 12,188M (Table 1)" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§2 Long context, §5.3" } }, { "name": { "value": "27B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 27.016, "provenance": "derived", "note": "417M vision + 1,416M embedding + 25,600M non-embedding = 27,016M (Table 1)" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§2 Long context, §5.3" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "All layers have dense FFNs; no routing anywhere in the report; Table 5 lists Gemma-3-27B-IT as 'Dense'" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "block_type dense" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "block_type dense" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "block_type dense" }, "routing": { "value": null, "provenance": "n/a", "ref": "block_type dense" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "block_type dense" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "block_type dense" } }, "attention_variant": { "value": "sliding-window", "provenance": "stated", "ref": "§1, §2", "note": "GQA-based: local layers are sliding-window self-attention with a 1024-token span; global layers are full-length self-attention; the report's headline mechanism is the 5:1 local/global interleaving" }, "attention_layer_pattern": { "value": "5:1 interleaving: 5 local sliding-window layers (span 1024 tokens) per 1 global layer, starting with a local layer as the first layer of the model", "provenance": "stated", "ref": "§2 ('We alternate between a local sliding window self-attention and global self-attention, with a pattern of 5 local layers for every global layer, starting with a local layer as the first layer of the model'); §1" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard residual connections per decoder layer; no attention over prior layer outputs described" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2 Long context ('We increase RoPE base frequency from 10k to 1M on global self-attention layers')" }, "rope_base": { "value": 1000000, "provenance": "stated", "ref": "§2, §5.3", "note": "Global self-attention layers: base raised from 10k (Gemma 2) to 1M; local layers keep 10k" }, "partial_rope": { "value": false, "provenance": "inferred", "note": "Absence rule: no partial/rotary-subset RoPE described" }, "extension": { "method": { "value": "PI", "provenance": "stated", "ref": "§5.3 ('we pre-train our models with 32K sequences and then scale the 4B, 12B, and 27B models up to 128K tokens at the end of pre-training while rescaling RoPE [Chen et al., 2023]'); §2 ('We follow a process similar to the positional interpolation of Chen et al. [2023] to extend the span of the global self-attention layers')" }, "factor": { "value": 8, "provenance": "stated", "ref": "§5.3 ('We find a scaling factor of 8 to work well in practice')" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§2 Long context ('Gemma 3 models support context length of 128K tokens, with the exception of the 1B model that has 32K')" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2 ('Grouped-Query Attention (GQA) with post-norm and pre-norm with RMSNorm')" }, "placement": { "value": "mixed", "provenance": "stated", "ref": "§2 ('with post-norm and pre-norm with RMSNorm' — both pre- and post-norm applied)" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2 ('Inspired by Dehghani et al. [2023], Wortsman et al. [2023] and Chameleon Team [2024], we replace the soft-capping of Gemma 2 with QK-norm.')", "note": "Architecture-wide replacement of Gemma 2's soft-capping with QK-norm on the attention; the report describes it as applying to the model generally, not only to one layer type" } }, "activation": { "value": null, "provenance": "unknown", "note": "Report never names the FFN activation (it only says 'most architecture elements similar to the first two Gemma versions', §2)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "stated", "ref": "§2 ('we replace the soft-capping of Gemma 2 with QK-norm')", "note": "Gemma 2's logit soft-capping is explicitly removed in Gemma 3" }, "other": { "value": [], "provenance": "inferred", "note": "No other stability mechanisms described beyond QK-norm (captured in normalization.qk_norm)" } } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§5.3 ('pre-train our models with 32K sequences and then scale the 4B, 12B, and 27B models up to 128K tokens at the end of pre-training')", "note": "Reference 27B: 32K pre-training then rescaled to 128K; the 1B variant stays at 32K" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§2 Long context ('Gemma 3 models support context length of 128K tokens')" }, "extension_method": { "value": "RoPE rescaling (positional-interpolation-style, factor 8) applied at the end of pre-training to extend the 32K-trained 4B/12B/27B models to 128K; global-layer RoPE base raised 10k → 1M while local layers stay at 10k", "provenance": "stated", "ref": "§5.3, §2" } }, "tokenizer": { "algorithm": { "value": "SentencePiece", "provenance": "stated", "ref": "§2.2 Tokenizer ('a SentencePiece tokenizer with split digits, preserved whitespace, and byte-level encodings')" }, "notes": { "value": "Same tokenizer as Gemini 2.0; ~262k entries, more balanced for non-English languages; PT and IT models share the tokenizer, with control tokens reserved for IT formatting (/); a [BOS] token must be added explicitly (add_bos=True); PT models end generation with , IT models with ", "provenance": "stated", "ref": "§2.2, §3, Table 4" } }, "training": { "tokens": { "value": 14, "provenance": "stated", "ref": "§2.2 Pre-training ('we train on 14T tokens for Gemma 3 27B, 12T for the 12B version, 4T for the 4B, and 2T tokens for the 1B')", "note": "Reference 27B; the token budget increase over Gemma 2 accounts for the mix of images and text" }, "data_composition": { "value": "Slightly larger token budget than Gemma 2; mixture of images and text; increased multilingual data (both monolingual and parallel) with language-imbalance handling inspired by Unimax (Chung et al. 2023); filtering to reduce unsafe/unwanted utterances and personal information, decontamination of evaluation sets, recitation-risk reduction, and a quality reweighting step (Sachdeva et al. 2024)", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "Pre-train at 32K sequences, then rescale the 4B/12B/27B models to 128K at the end of pre-training via RoPE rescaling (factor 8); knowledge distillation during pre-training; per-size token budgets (2T/1B, 4T/4B, 12T/12B, 14T/27B)", "provenance": "stated", "ref": "§5.3, §2.2" }, "optimizer": { "value": null, "provenance": "unknown", "note": "No optimizer named; only ZeRO-3 optimizer-state sharding is described (§2.4)" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "No batch sizes disclosed; Table 2 gives only sharding counts (data / sequence / replica) per model" }, "precision": { "value": null, "provenance": "unknown", "note": "Training precision not stated; Table 3 memory footprints are for 'raw (bfloat16)' checkpoints" }, "parallelism": { "value": "ZeRO-3 optimizer-state sharding; data-replica reduction over the data center network (Pathways) for multi-pod training; sharding by data, sequence, and replica (Table 2); JAX/Pathways single-controller paradigm with the GSPMD partitioner and the MegaScale XLA compiler", "provenance": "stated", "ref": "§2.4, Table 2" }, "hardware": { "value": "TPUv4, TPUv5e, and TPUv5p (1B: 512 TPUv5e chips; 4B: 2048 TPUv5e; 12B: 6144 TPUv4; 27B: 6144 TPUv5p); vision embeddings pre-computed so the vision encoder adds no training cost", "provenance": "stated", "ref": "§2.4, Table 2" }, "compute": { "value": null, "provenance": "unknown", "note": "No FLOPs or TPU-hours disclosed; per-variant token budgets are given (§2.2)" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3 ('Pre-trained models are turned into instruction-tuned models with an improved post-training approach'); §7.2 ('we also use both SFT and RLHF')" }, "preference_optimization": { "value": "RLHF", "provenance": "stated", "ref": "§7.2 ('both SFT and RLHF'); §3", "note": "RL fine-tuning phase based on improved versions of BOND, WARM, and WARP; rewards from weight-averaged reward models trained on human feedback, code execution feedback, and ground-truth rewards for math problem solving" }, "reasoning_training": { "value": "RL objectives cover helpfulness, math (ground-truth rewards for solving math problems), coding (code execution feedback), reasoning, instruction-following, and multilingual abilities; no long-CoT / RLVR pipeline is described", "provenance": "stated", "ref": "§3" }, "distillation": { "value": "All Gemma 3 models are trained with knowledge distillation (256 logits per token sampled, weighted by teacher probabilities, cross-entropy on the renormalized teacher distribution); IT models use an improved distillation from a large IT teacher; the teacher model is never named", "provenance": "stated", "ref": "§2.2 Distillation, §3, Abstract ('The Gemma 3 models are trained with distillation')" } }, "modality": { "type": { "value": "multimodal", "provenance": "stated", "ref": "§1 ('most Gemma 3 models are compatible with a tailored version of the SigLIP vision encoder')", "note": "Reference 27B is multimodal (text + image); the 1B variant is text-only (Table 1 lists 0 vision-encoder parameters for 1B)" }, "attachment": { "value": "adapter", "provenance": "stated", "ref": "§1, §2.1, §5.5", "note": "Frozen 400M SigLIP encoder (ViT, 896×896 input, 4×4 average pooling) shared across 4B/12B/27B; images become 256 soft tokens fed to the LM; Pan & Scan (P&S) inference-time adaptive windowing for non-square/high-resolution images, inspired by LLaVA" } }, "efficiency": { "kv_cache": { "value": "5:1 local:global interleaving with a 1024-token local span bounds each local layer's KV cache at 1024 tokens while only global layers attend the full context; report measures KV-cache memory overhead falling from ~60% (global-only) to <15% (5:1, sw=1024) of model memory at 32K prefill", "provenance": "stated", "ref": "§1, §5.2 (Figs. 5, 6)" }, "quantization": { "value": "QAT (typically ~5,000 finetuning steps, targeting llama.cpp-style engines) producing per-channel int4, per-block int4 (blocks=32), and switched fp8 weight representations; Table 3 reports memory footprints (e.g. 27B: 54.0 GB bf16 / 14.1 GB int4 / 15.3 GB int4_blocks=32 / 27.4 GB SFP8, +KV at 32K context)", "provenance": "stated", "ref": "§2.3, Table 3" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 67.5, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "LiveCodeBench", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 29.7, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Bird-SQL (dev)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 54.4, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GPQA Diamond", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 42.4, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SimpleQA", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 10.0, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "FACTS Grounding", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 74.9, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Global MMLU-Lite", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 75.1, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MATH", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 89.0, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "HiddenMath", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 60.3, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMMU (val)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 64.9, "provenance": "stated", "ref": "Table 6 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Chatbot Arena (Elo)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 1338, "provenance": "stated", "ref": "Table 5 (Gemma-3-27B-IT; preliminary results of March 8, 2025)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "RULER (128K)", "provenance": "stated", "ref": "Table 15" }, "value": { "value": 66.0, "provenance": "stated", "ref": "Table 15 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 15", "provenance": "stated", "ref": "Table 15" } }, { "name": { "value": "MRCR (128K)", "provenance": "stated", "ref": "Table 15" }, "value": { "value": 59.3, "provenance": "stated", "ref": "Table 15 (Gemma 3 27B IT column)" }, "ref": { "value": "Table 15", "provenance": "stated", "ref": "Table 15" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Multimodal addition to the Gemma family: a tailored 400M SigLIP vision encoder shared across 4B/12B/27B, images condensed to a fixed 256 soft tokens, with a Pan & Scan adaptive windowing algorithm for native-aspect-ratio, high-resolution images", "provenance": "stated", "ref": "§1, §2.1" }, "ref": { "value": "§1, §2.1", "provenance": "stated", "ref": "§1, §2.1" } }, { "text": { "value": "KV-cache reduction for long context: 5:1 interleaving of local (sliding-window, 1024-token span) and global attention layers, so only global layers attend the 128K context", "provenance": "stated", "ref": "§1, §2" }, "ref": { "value": "§1, §2", "provenance": "stated", "ref": "§1, §2" } }, { "text": { "value": "Replacing Gemma 2's soft-capping with QK-norm (inspired by Dehghani et al. 2023, Wortsman et al. 2023, Chameleon Team 2024)", "provenance": "stated", "ref": "§2" }, "ref": { "value": "§2", "provenance": "stated", "ref": "§2" } }, { "text": { "value": "A novel post-training recipe (improved knowledge distillation from a large IT teacher plus RL fine-tuning based on BOND, WARM, and WARP) that significantly improves math, chat, instruction-following, and multilingual abilities", "provenance": "stated", "ref": "§1, §3" }, "ref": { "value": "§1, §3", "provenance": "stated", "ref": "§1, §3" } }, { "text": { "value": "Multilinguality: wider language coverage via increased monolingual and parallel data with Unimax-style imbalance handling; Global MMLU-Lite 75.1 and WMT24++ 53.4 on the 27B IT model", "provenance": "stated", "ref": "§1, §2.2, Table 18" }, "ref": { "value": "§1, §2.2, Table 18", "provenance": "stated", "ref": "§1, §2.2, Table 18" } }, { "text": { "value": "Scale-relative performance: Gemma3-4B-IT competitive with Gemma2-27B-IT, and Gemma3-27B-IT comparable to Gemini-1.5-Pro across benchmarks", "provenance": "stated", "ref": "Abstract, §4" }, "ref": { "value": "Abstract, §4", "provenance": "stated", "ref": "Abstract, §4" } } ], "assessment": { "value": "The block is a conservative evolution of the dense GQA decoder-only skeleton shared with meta-llama-3.1/meta-llama-3 (RMSNorm pre/post-norm, RoPE, SwiGLU-class FFN by lineage). The genuinely new axis for the atlas is the per-layer attention pattern: a 5:1 local/global interleaving with a 1024-token sliding-window span (the first sliding-window record in the atlas; phi-3-small's blocksparse pattern is the nearest peer but is a different mechanism) and the dual RoPE bases (10k local / 1M global). QK-norm replacing soft-capping follows deepseek-v3's QK-norm practice but is applied to a dense GQA model. Long context reuses positional interpolation (factor 8) rather than a new mechanism. Vision is an adapter (frozen SigLIP + P&S), not native. The 'novel post-training recipe' is a training contribution (distillation from an unnamed teacher + BOND/WARM/WARP-based RL), and the report's flagship claim (4B-IT ≈ Gemma2-27B-IT; 27B-IT ≈ Gemini-1.5-Pro) is internal to the report and not independently verifiable. The report is unusually sparse on architecture disclosure: no layer counts, hidden dims, FFN widths, or head counts for any variant.", "provenance": "inferred", "note": "comparison against atlas entries (meta-llama-3.1, meta-llama-3, deepseek-v3, microsoft-phi-3, mistral-mixtral-8x7b) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "All layers have dense FFNs; no routing anywhere (Table 5 lists the model as 'Dense')" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "5 of every 6 layers (≈83%) are local sliding-window attention (span 1024); 1 of 6 global full attention; ≥80% majority rule" }, "modality": { "value": "multimodal", "provenance": "derived", "note": "From modality.type: reference 27B is multimodal (text + image via SigLIP adapter); the 1B variant is text-only" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true ('We release all our models to the community'); data mixture described but not released" }, "scale_class": { "value": "large", "provenance": "derived", "note": "Reference 27B: 10B ≤ 27B < 100B" }, "design_intent": { "value": "on-device", "provenance": "inferred", "note": "Framed as lightweight open models 'designed to run on standard consumer-grade hardware such as phones, laptops, and high-end GPUs' (§1); the 27B reference sits at the high-end-GPU end of that range. Secondary intents: multimodal and long-context (128K), noted in prose." } }, "prose": { "design_overview": { "value": "Gemma 3 is Google DeepMind's third-generation open decoder-only transformer family (1B–27B), and its headline architectural change targets long-context inference cost: the uniform attention stack of Gemma 2 gives way to a 5:1 interleaving of local and global layers, starting with a local layer, where each local layer is a sliding-window self-attention with a span of only 1024 tokens and each global layer keeps full-length attention. Only the global layers therefore attend over the full 128K context, cutting KV-cache memory overhead from roughly 60% of model memory (global-only) to under 15% at 32K prefill. Attention remains grouped-query (GQA) with RMSNorm pre- and post-norm, and Gemma 2's logit soft-capping is replaced by QK-norm, borrowed from the ViT and Chameleon lines. Long context is reached by pre-training at 32K and rescaling RoPE at the end of pre-training (positional-interpolation-style factor 8), with the global layers' RoPE base raised from 10k to 1M while local layers stay at 10k. Multimodality is an adapter: a frozen 400M SigLIP encoder (896×896, shared across 4B/12B/27B) converts images into 256 soft tokens with 4×4 average pooling, plus an inference-time Pan & Scan windowing scheme for arbitrary aspect ratios; the 1B variant is text-only. All models are trained with knowledge distillation (teacher unnamed) on 2T–14T tokens, and the IT models use an improved post-training recipe built on distillation and BOND/WARM/WARP-style RL with math and code-execution rewards. The report is unusually light on architecture numbers: it discloses no layer counts, hidden dims, FFN widths, or head counts for any variant.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Grouped-Query Attention (GQA) — KV sharing retained for all attention layers", "provenance": "stated", "ref": "§2 (cites Ainslie et al. [2023])" }, "origin": { "value": "GQA (Ainslie et al., 2023)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Sliding-window local attention — 1024-token span on local layers", "provenance": "stated", "ref": "§2 (cites Beltagy et al. [2020], Longformer)" }, "origin": { "value": "Longformer (Beltagy et al., 2020)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "QK-norm replacing soft-capping", "provenance": "stated", "ref": "§2 (cites Dehghani et al. [2023], Wortsman et al. [2023], Chameleon Team [2024])" }, "origin": { "value": "Dehghani et al. 2023 / Wortsman et al. 2023 / Chameleon (not in atlas)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Positional interpolation (RoPE rescaling, factor 8) for context extension", "provenance": "stated", "ref": "§2, §5.3 (cites Chen et al. [2023])" }, "origin": { "value": "Chen et al. 2023 (positional interpolation)", "provenance": "stated", "ref": "§2, §5.3" } }, { "mechanism": { "value": "SigLIP vision encoder (400M, frozen) with LLaVA-inspired flexible-resolution Pan & Scan", "provenance": "stated", "ref": "§1, §2.1 (cites Zhai et al. [2023], Liu et al. [2024])" }, "origin": { "value": "SigLIP (Zhai et al., 2023) / LLaVA (Liu et al., 2024) (not in atlas)", "provenance": "stated", "ref": "§1, §2.1" } }, { "mechanism": { "value": "Knowledge distillation for pre-training and instruction tuning (256 sampled logits per token)", "provenance": "stated", "ref": "§2.2, §3 (cites Hinton et al. [2015])" }, "origin": { "value": "Hinton et al. 2015 (knowledge distillation)", "provenance": "stated", "ref": "§2.2" } } ], "influence_out": [ { "model_slug": { "value": "jetbrains-mellum2-thinking-12b-a2-5b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Layer-selective context scaling — extension applied only to global-attention layers ('first reported in the Gemma 3 technical report… subsequently adopted by OLMo 3'); Mellum 2 uses YaRN as the scaling method", "provenance": "inferred", "note": "mechanism as stated in jetbrains-mellum2-thinking-12b-a2-5b report's lineage_in" } } ], "notable_omissions": { "value": [ "No layer counts, hidden dimensions, FFN widths, or attention head counts for any variant (no configuration table anywhere in the report; Table 1 gives parameter counts only)", "FFN activation function never named (Gemma 2 lineage implies GeGLU but the report does not state it)", "Vocab size printed inconsistently: 262k entries in §2.2 vs 256k in the Table 1 caption", "No training optimizer, learning-rate schedule, batch sizes, or training precision disclosed", "No training compute (FLOPs or TPU-hours) disclosed", "No license named for the model weights (arXiv page's CC BY 4.0 covers the paper only)", "The distillation teacher (pre-training and IT) is never identified", "No per-variant architecture table (e.g. which layers are local/global per size, or per-size RoPE bases)", "Input/output embedding tying not disclosed", "Only the 1B–27B sizes appear in this report; a sub-1B (270M-class) variant is not covered" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "google-gemma-3.json" }, { "slug": { "value": "google-gemma-4", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2607.02770", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/google-gemma-4/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Gemma 4", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "E2B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "E4B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "12B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "26B-A4B", "provenance": "stated", "ref": "Abstract, Table 1" }, { "value": "31B", "provenance": "stated", "ref": "Abstract, Table 1" } ], "org": { "value": "Google DeepMind (Gemma Team)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-07-24", "provenance": "stated", "ref": "Header: 'arXiv:2607.02770v2 [cs.CL] 24 Jul 2026'", "note": "Date printed on the report (v2 header); arXiv API shows v1 published 2026-07-02 (2607.02770)" }, "report_url": { "value": "https://arxiv.org/abs/2607.02770", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "§1 ('We release the Gemma 4 models under an Apache 2.0 license')", "note": "Model weights license; the paper itself is CC BY 4.0 per the arXiv page" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('open-weight, natively multimodal language models'); §1 (Apache 2.0 release)" } }, "scale": { "total_params": { "value": 31.25, "provenance": "derived", "note": "Table 1 components for the 31B reference: 550M vision encoder + 1,410M embedder + 29,290M einsums = 31,250M; report labels the model '31B'. The 500M MTP drafter is a separate speculative-decoding head and is excluded (report's own labels exclude it too)." }, "active_params_per_token": { "value": 31.25, "provenance": "derived", "note": "Dense model: active = total (31,250M); Table 4 lists Gemma 4 31B as 'Dense' with #params/#activated 31B" }, "layers": { "value": null, "provenance": "unknown", "note": "No layer count disclosed for any LLM backbone; only the vision encoders are dimensioned (Table 10: 27 layers for the 550M encoder, 16 for the 150M) and the MTP drafter (4 layers, §2.6)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed for any LLM backbone (Table 10 gives d_model 1152/768 for the vision encoders only)" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed (d_MLP 4304/3072 in Table 10 belong to the vision encoders)" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "Cannot derive without ffn_inner_dim and hidden_dim" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Query head count not disclosed (Table 10 heads 16/12 are the vision encoders)" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Report never names GQA/MQA; the 'KV cache sharing [Shazeer, 2019]' at ratios 20/35 (E2B) and 18/42 (E4B) is cross-layer cache sharing, not a KV-head count" }, "head_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed; cannot derive without hidden_dim and head counts" }, "vocab_size": { "value": 262000, "provenance": "stated", "ref": "§2.4 Tokenizer ('The vocabulary has 262k entries'); Table 1 caption ('The vocabulary we use has 262k entries')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "variants": [ { "name": { "value": "E2B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 5.065, "provenance": "derived", "note": "305M audio + 150M vision + 400M embedder + 2,340M per-layer embeddings + 1,870M einsums = 5,065M (Table 1); report: '5B total', 'effective 2.3B' (per-layer embeddings excluded from effective count, as in Gemma 3n); 76M drafter excluded" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "No per-variant context window stated; Table 9 evaluates E2B at 32k/128k only (dash at ~256k)" } }, { "name": { "value": "E4B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 7.885, "provenance": "derived", "note": "305M audio + 150M vision + 670M embedder + 2,820M per-layer embeddings + 3,940M einsums = 7,885M (Table 1); report: '8B total', 'effective 4.5B'; 77M drafter excluded" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "No per-variant context window stated; Table 9 evaluates E4B at 32k/128k only (dash at ~256k)" } }, { "name": { "value": "12B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 11.89, "provenance": "derived", "note": "1,000M embedder + 10,890M einsums = 11,890M (Table 1); encoder-free (no vision/audio encoder columns); report labels '12B'; 400M drafter excluded" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "No per-variant context window stated; Table 9 evaluates 12B at 32k/128k/~256k" } }, { "name": { "value": "26B-A4B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 25.79, "provenance": "derived", "note": "550M vision + 740M embedder + 24,500M einsums = 25,790M (Table 1); report labels '26B total'; 430M drafter excluded. Table 1 prints active einsums 2,800M; text §2 says '3.8B activated'; Table 4 prints '26B / 4B'" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "No per-variant context window stated; Table 9 evaluates 26B-A4B at 32k/128k/~256k" } }, { "name": { "value": "31B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 31.25, "provenance": "derived", "note": "550M vision + 1,410M embedder + 29,290M einsums = 31,250M (Table 1); report labels '31B'; 500M drafter excluded" }, "layers": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "No per-variant context window stated; Table 9 evaluates 31B at 32k/128k/~256k" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "Reference 31B is dense (Table 4 lists 'Dense'); the family also ships a sparse-MoE variant 26B-A4B whose routing is undisclosed; per-layer density not reported for either" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§2 (Dense and MoE paragraph)", "note": "Reference 31B is dense; the family's MoE variant (26B-A4B, 3.8B activated / 26B total) has no expert count, top-k, or router disclosed anywhere in the report" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§2 (Dense and MoE paragraph)", "note": "Reference 31B is dense; no routing details disclosed for 26B-A4B" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§2 (Dense and MoE paragraph)", "note": "Reference 31B is dense; no MoE internals disclosed" }, "routing": { "value": null, "provenance": "n/a", "ref": "§2 (Dense and MoE paragraph)", "note": "Reference 31B is dense; no router described for 26B-A4B" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§2 (Dense and MoE paragraph)", "note": "Reference 31B is dense; no load-balancing scheme disclosed" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§2 (Dense and MoE paragraph)", "note": "Reference 31B is dense; expert granularity of 26B-A4B undisclosed" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§1, §2 (Long-context efficiency: '5:1 ratio of local sliding window to global self-attention (4:1 for the 2.3B model)')" }, "attention_layer_pattern": { "value": "5:1 interleaving of local sliding-window attention layers per global self-attention layer for all models except E2B (4:1); local window span not disclosed; global layers reuse keys as values (values=keys) except in E2B and E4B; KV cache shared across layers at ratios 20/35 (E2B) and 18/42 (E4B)", "provenance": "stated", "ref": "§2 (Long-context efficiency)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard decoder-only residual stack; no attention over prior layer outputs or learned connection matrix described" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2 (Long-context efficiency)", "note": "pp-RoPE with p=0.25 on global attention layers and plain RoPE on local attention layers; vision encoders additionally use axial 2D-RoPE with 2D absolute positional embeddings (§2.1)" }, "rope_base": { "value": 1000000, "provenance": "stated", "ref": "§2 ('The RoPE frequencies are set to 1M and 10k on global and local attention layers, respectively')", "note": "1M on global layers, 10k on local layers (dual bases, as in Gemma 3)" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§2 ('We encode position with pp-RoPE with p=0.25 on global attention layers')", "note": "pp-RoPE (partial rotary) with p=0.25 on global layers; local layers use plain RoPE" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "No context-extension method disclosed (no YaRN/NTK/interpolation); long context is served by the 1M global RoPE base, but the report never states training vs deployed lengths" }, "factor": { "value": null, "provenance": "unknown", "note": "No extension factor disclosed" }, "target_length": { "value": null, "provenance": "unknown", "note": "Deployed context length never stated; Table 9 evaluates up to ~256k, Table 3 footprints at 32k" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2 ('Our models have pre-norm and post-norm with RMSNorm')" }, "placement": { "value": "mixed", "provenance": "stated", "ref": "§2 ('pre-norm and post-norm with RMSNorm')" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2 ('and QKNorm [Henry et al., 2020]')", "note": "QK-norm stated as part of the model architecture (Gemma 3 replaced soft-capping with QK-norm; Gemma 4 keeps it)" } }, "activation": { "value": null, "provenance": "unknown", "note": "FFN activation never named in the report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: report never mentions logit softcapping (Gemma 3 removed it in favor of QK-norm; Gemma 4 is silent)" }, "other": { "value": [ "A scalar scale at each block bounds activation ranges to fit fp16 for stable inference (§2.5 QAT section)" ], "provenance": "stated", "ref": "§2.5" } } }, "context": { "trained_length": { "value": null, "provenance": "unknown", "note": "Training context length never stated" }, "deployed_length": { "value": null, "provenance": "unknown", "note": "Not stated; long-context evals reach 32k/128k/~256k (Table 9), memory footprints are reported at 32k (Table 3)" }, "extension_method": { "value": null, "provenance": "unknown", "note": "No extension method disclosed; RoPE bases 1M (global) / 10k (local) are stated (§2)" } }, "tokenizer": { "algorithm": { "value": "SentencePiece", "provenance": "stated", "ref": "§2.4 Tokenizer ('a SentencePiece tokenizer with split digits, preserved whitespace, and byte-level encodings')" }, "notes": { "value": "Same tokenizer as Gemini Team [2025]; 262k vocabulary; IT formatting adds control tokens (<|think|>, <|turn|>, <|tool_call|>, <|channel>) for thinking mode and function calling (Table 11)", "provenance": "stated", "ref": "§2.4, §3 (PT vs IT formatting), Table 11" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "No pre-training token count disclosed" }, "data_composition": { "value": "Large-scale, diverse collection from a wide range of domains and modalities, including web documents, code, images, and audio (audio for E2B, E4B and 12B); data cutoff date January 2025; filtered to decontaminate benchmarks and reduce unwanted or unsafe utterances and recitation risk", "provenance": "stated", "ref": "§2.4" }, "curriculum": { "value": "none disclosed", "provenance": "stated", "ref": "§2.4 ('We follow a similar pre-training as Gemma 3')", "note": "No staging, annealing, or context curriculum described" }, "optimizer": { "value": null, "provenance": "unknown", "note": "Optimizer never named (only ZeRO-3 state sharding is described, §2.7)" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "Not disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "Master training precision not stated; Table 3 labels raw checkpoints 'bf16'; QAT weight formats (mobile int2/int4 mix, Q4_0) disclosed in §2.5" }, "parallelism": { "value": "ZeRO-3 optimizer-state sharding; Pathways data-replica reduction over the data-center network; GSPMD partitioner with the MegaScale XLA compiler; Slice-Granularity Elasticity for failure recovery on the larger models; sharding by data, sequence, and replica (Table 2)", "provenance": "stated", "ref": "§2.7, Table 2" }, "hardware": { "value": "TPUv4 and TPUv6e: E2B 4,096 v6e chips; E4B 6,144 v6e; 12B 12,288 v4; 26B-A4B 6,144 v6e; 31B 10,240 v6e", "provenance": "stated", "ref": "§2.7, Table 2" }, "compute": { "value": null, "provenance": "unknown", "note": "No FLOPs or TPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "inferred", "note": "Instruction tuning stated: 'Pre-trained models are turned into instruction-tuned models with a similar post-training approach as in Gemma 3' (§3); SFT specifically is not named in this report" }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "§3", "note": "No RL method named; post-training is deferred to 'a similar post-training approach as in Gemma 3' without details" }, "reasoning_training": { "value": "Thinking mode: models generate a reasoning trace before responding (o1-style, cited OpenAI [2024]); activated via the <|think|> token in a leading system turn (Table 11); the training recipe behind the thinking mode is not disclosed", "provenance": "stated", "ref": "§1, §3, Table 11" }, "distillation": { "value": null, "provenance": "unknown", "note": "Distillation never mentioned in this report (Gemma 3's report described distillation explicitly; Gemma 4 is silent)" } }, "modality": { "type": { "value": "multimodal", "provenance": "stated", "ref": "Abstract ('natively multimodal'); §1 ('capable of seamlessly processing text, images, and audio')", "note": "Reference 31B is multimodal via vision only; audio encoders exist only on E2B/E4B (305M) and the encoder-free 12B; 31B and 26B-A4B have no audio input (Table 1)" }, "attachment": { "value": "adapter", "provenance": "stated", "ref": "§1 ('Gemma 4 models have frozen vision and audio encoders'), §2.1", "note": "31B reference attaches a frozen 550M ViT vision encoder (patch 16, axial 2D-RoPE with non-causal attention plus 2D absolute positional embeddings; variable aspect ratios; Nmax in {70,140,280,560,1120}); encoders are frozen during pre-training. The 12B is encoder-free: raw 40ms audio chunks at 16kHz (640-dim vectors) and 48x48x3 image patches via a single 35M matmul are projected directly into the LLM embedding space (§2.3)" } }, "efficiency": { "kv_cache": { "value": "Local sliding-window layers bound KV growth to the window span; global layers reuse keys as values (values=keys, except E2B/E4B) and use pp-RoPE p=0.25, reducing the global KV cache footprint by up to 37.5%; KV caches shared across layers at ratios 20/35 (E2B) and 18/42 (E4B); int8 KV caching offered (Table 3, '+KV' at 32k context)", "provenance": "stated", "ref": "§1, §2, Table 3" }, "quantization": { "value": "QAT-trained quantized checkpoints: mobile quantization (per-channel low-bitwidth weights, int2/int4 mix, with int8 activation quantization) and Q4_0 blockwise quantization; raw checkpoints bf16; QAT also applied to encoders — 150M vision encoder W8A8 (forward memory 400→200 MB), audio encoder int8 activations with {2,4,8}-bit weights by layer cluster (on-disk 390→87 MB)", "provenance": "stated", "ref": "§2.5, Table 3" }, "speculative_dedup": { "value": "Autoregressive MTP drafter head for speculative decoding: a 4-layer Transformer block (d_model 256 for E2B/E4B, 1024 for 26B-A4B/31B; 3 local + 1 global attention layers) with a separate embedder, cross-attending to the main model's KVs — no MTP prefill, any draft length; E2B/E4B drafters use cluster top-k decoding (d×262,000 → d×4,096)", "provenance": "stated", "ref": "§1, §2.6" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "Report-wide absence (no serving stack described)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "Arena Text Elo", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 1451, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU Pro", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 85.2, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "AIME 2026 (no tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 89.2, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LiveCodeBench v6", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 80.0, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Codeforces Elo", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 2150, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "GPQA Diamond", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 84.3, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HLE", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 19.5, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MMMU Pro (1120 vision tokens)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 76.9, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MATH-Vision (1120 vision tokens)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 85.6, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "InfographicVQA (1120 vision tokens)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 92.0, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "RULER accuracy 128k", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 96.4, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "LOFT Text Retrieval Recall@k 128k", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 79.5, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Thinking mode: models output a reasoning trace before responding, improving reasoning-heavy domains such as mathematics and coding", "provenance": "stated", "ref": "§1, §3" }, "ref": { "value": "§1, §3", "provenance": "stated", "ref": "§1, §3" } }, { "text": { "value": "Long-context efficiency: 5:1 local sliding-window to global attention (4:1 for E2B), pp-RoPE positional encoding, KV cache sharing, and keys-as-values reuse in global layers, reducing the global KV cache footprint by up to 37.5%", "provenance": "stated", "ref": "§1, §2" }, "ref": { "value": "§1, §2", "provenance": "stated", "ref": "§1, §2" } }, { "text": { "value": "An autoregressive MTP drafter head for speculative decoding that eliminates MTP prefill and supports any draft length", "provenance": "stated", "ref": "§1, §2.6" }, "ref": { "value": "§1, §2.6", "provenance": "stated", "ref": "§1, §2.6" } }, { "text": { "value": "Quantized checkpoints trained with quantization-aware training (mobile int2/int4 and Q4_0 formats) with minimal quality impact", "provenance": "stated", "ref": "§1, §2.5" }, "ref": { "value": "§1, §2.5", "provenance": "stated", "ref": "§1, §2.5" } }, { "text": { "value": "A unified, encoder-free architecture for the 12B model that projects raw 40ms audio chunks and image patches into the LLM embedding space, removing separate encoders and reducing memory fragmentation", "provenance": "stated", "ref": "§1, §2.3" }, "ref": { "value": "§1, §2.3", "provenance": "stated", "ref": "§1, §2.3" } }, { "text": { "value": "Gemma 4 31B is the leading dense open model on the Arena Text leaderboard, rivaling much larger open models in human-rated tasks", "provenance": "stated", "ref": "§4.1, Table 4" }, "ref": { "value": "§4.1, Table 4", "provenance": "stated", "ref": "§4.1, Table 4" } } ], "assessment": { "value": "A conservative, efficiency-focused evolution of the google-gemma-3 skeleton: same dense GQA-class decoder with 5:1 local/global attention (4:1 on E2B), dual RoPE bases, RMSNorm pre/post-norm + QK-norm, and the Gemini/SentencePiece tokenizer. The genuinely new mechanisms for the atlas are incremental rather than structural: keys-as-values reuse on global layers (values=keys, from Kayyam et al. 2026), cross-layer KV cache sharing (Shazeer 2019 MQA lineage), pp-RoPE with p=0.25 partial rotation on global layers (Barbero et al. 2025), per-layer embeddings on the small dense models (Gemma 3n lineage, not an atlas record), an EAGLE-style MTP drafter (deepseek-v3's MTP is the nearest atlas peer), QAT-shipped quantized checkpoints, and o1-style thinking mode. The encoder-free 12B (raw audio chunks + image patches through a 35M matmul) is the most distinctive claim — no atlas peer ingests raw patches without an encoder. Notably, the report does not name GQA, the FFN activation, or any MoE routing detail for 26B-A4B, and audio input is absent from the 31B and 26B-A4B variants despite the 'natively multimodal' framing. No new attention or mixing family is introduced.", "provenance": "inferred", "note": "comparison against atlas entries (google-gemma-3, deepseek-v3, meta-llama-3.1, moonshot-kimi-k3, zai-glm-4-5) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "Reference 31B is dense (Table 4 'Dense'); the family also includes sparse-MoE 26B-A4B, but the majority rule applies to the reference variant's layers" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "5 of every 6 layers (≈83%) are local sliding-window attention per the 5:1 pattern on the reference 31B; ≥80% majority rule (E2B is 4:1 = 80%); same classification as google-gemma-3" }, "modality": { "value": "multimodal", "provenance": "derived", "note": "From modality.type: reference 31B is multimodal (text + image via frozen ViT adapter); audio only on E2B/E4B/12B" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Apache 2.0 release, §1); training data mixture described at a high level but not released" }, "scale_class": { "value": "large", "provenance": "derived", "note": "Reference 31B: 31.25B, so 10B ≤ 31.25B < 100B" }, "design_intent": { "value": "on-device", "provenance": "inferred", "note": "Framed as built 'to serve a variety of on-device hardware' (§1) with mobile quantization and per-layer embeddings for memory efficiency; secondary intents: reasoning (thinking mode) and long context, noted in prose" } }, "prose": { "design_overview": { "value": "Gemma 4 is the fourth generation of Google DeepMind's open Gemma family, a suite of five natively multimodal decoder-only Transformers spanning dense 31B, 12B, and the per-layer-embedding E4B/E2B models plus one sparse MoE (26B total, 3.8B active), all sharing one SentencePiece tokenizer with 262k entries. Architecturally it is a direct refinement of Gemma 3: the backbone keeps the 5:1 local sliding-window to global attention interleaving (4:1 on E2B) with dual RoPE bases (10k local, 1M global), but adds pp-RoPE with p=0.25 on global layers, reuses keys as values in global attention (values=keys, except E2B/E4B), and shares KV caches across layers at 20/35 or 18/42 ratios — together cutting the global KV cache by up to 37.5%. Blocks keep RMSNorm pre- and post-norm with QK-norm. Vision attaches through frozen ViT encoders (550M for 31B/26B-A4B, 150M for E2B/E4B) with axial 2D-RoPE and aspect-ratio-preserving resizing; audio exists only on E2B/E4B via a slimmed 305M USM-style Conformer encoder, while the flagship novelty is the 12B's encoder-free design, which feeds raw 40ms audio chunks and 48×48×3 image patches through a 35M matmul directly into the LLM. Every model ships with an MTP drafter for speculative decoding and QAT-quantized checkpoints (mobile int2/int4, Q4_0). A thinking mode — reasoning traces toggled by a <|think|> token — is the family's headline behavioral feature, and Arena results put the 31B at the top of open dense models.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Local sliding-window / global attention interleaving (5:1, 4:1 for E2B) with dual RoPE bases (10k local / 1M global)", "provenance": "stated", "ref": "§2 ('Our local to global attention ratio patterns follow Gemma Team [2025a]')" }, "origin": { "value": "Gemma 3", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Per-layer embeddings on the small dense models (E2B/E4B)", "provenance": "stated", "ref": "§2 ('E2B and E4B use per-layer embeddings as in Gemma 3n')" }, "origin": { "value": "Gemma 3n", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "KV cache sharing across attention layers", "provenance": "stated", "ref": "§1, §2 (cites Shazeer [2019], multi-query attention lineage)" }, "origin": { "value": "Shazeer 2019 (MQA)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Keys-as-values reuse in global attention layers (values=keys)", "provenance": "stated", "ref": "§1, §2 (cites Kayyam et al. [2026])" }, "origin": { "value": "Kayyam et al. 2026", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "pp-RoPE (partial rotary) positional encoding with p=0.25 on global layers", "provenance": "stated", "ref": "§1, §2 (cites Barbero et al. [2025])" }, "origin": { "value": "Barbero et al. 2025 (pp-RoPE)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Autoregressive MTP drafter head for speculative decoding", "provenance": "stated", "ref": "§1, §2.6 (cites Li et al. [2024] EAGLE, Leviathan et al. [2023])" }, "origin": { "value": "EAGLE (Li et al. 2024)", "provenance": "stated", "ref": "§2.6" } }, { "mechanism": { "value": "USM-based audio encoder (two downsampling convolutions + twelve Conformer layers), slimmed from 680M to 305M", "provenance": "stated", "ref": "§2.2 (cites Zhang et al. [2023] USM, Gulati et al. [2020])" }, "origin": { "value": "Universal Speech Model (USM)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "SentencePiece tokenizer with split digits, preserved whitespace, and byte-level encodings", "provenance": "stated", "ref": "§2.4 ('We use the same tokenizer as Gemini Team [2025]')" }, "origin": { "value": "Gemini 2.5 (Gemini Team 2025)", "provenance": "stated", "ref": "§2.4" } } ], "influence_out": [], "notable_omissions": { "value": [ "No layer count, hidden dim, FFN dim, head count, or head dim for any LLM backbone (only the vision encoders are dimensioned, Table 10: 550M d_model 1152/d_MLP 4304/16 heads/27 layers; 150M 768/3072/12/16)", "FFN activation function never named", "MoE internals of 26B-A4B entirely undisclosed: no expert count, top-k, router, load balancing, or expert granularity", "Inconsistent active-parameter figures for 26B-A4B: §2 text says 3.8B activated, Table 1 prints 2,800M active einsums (≈4.1B with encoders+embedder), Table 4 prints '26B / 4B'", "No pre-training token count, FLOPs, or compute hours", "No optimizer, learning rate, or batch schedule disclosed (pre-training 'similar to Gemma 3', §2.4)", "Deployed context window never stated (Table 9 evaluates up to ~256k; Table 3 footprints at 32k); no context-extension method described", "Post-training recipe deferred to Gemma 3; no preference-optimization method named; thinking-mode training recipe not disclosed", "Distillation never mentioned (Gemma 3's report described distillation in detail)", "GQA/MQA never named — KV-head structure of the attention is undisclosed despite cross-layer KV sharing being a headline feature", "Input/output embedding tying not disclosed", "Audio input absent from the 31B and 26B-A4B variants, so 'natively multimodal' applies unevenly across the family (Table 1)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "google-gemma-4.json" }, { "slug": { "value": "inclusion-ling-2-6-1t", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2606.15079", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/inclusion-ling-2-6-1t/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Ling 2.6", "provenance": "stated", "ref": "Title ('Ling and Ring 2.6 Technical Report'); the report covers the Ling-2.6 and Ring-2.6 family" }, "variants": [ { "value": "Ling-2.6-flash", "provenance": "stated", "ref": "Abstract ('three open-sourced model weights: Ling-2.6-flash, Ling-2.6-1T, and Ring-2.6-1T')" }, { "value": "Ling-2.6-1T", "provenance": "stated", "ref": "Abstract; Table 1" }, { "value": "Ring-2.6-1T", "provenance": "stated", "ref": "Abstract; §3.2 (Ring-2.6 post-training from the shared base)" } ], "org": { "value": "Ling Team, Inclusion AI", "provenance": "stated", "ref": "Title byline" }, "release_date": { "value": "2026-06-13", "provenance": "stated", "ref": "Page 2 header: 'arXiv:2606.15079v1 [cs.CL] 13 Jun 2026'; title page also prints 'Date: June, 2026'" }, "report_url": { "value": "https://arxiv.org/abs/2606.15079", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named in the report (code and weight URLs are given, but no license string)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We open-source all checkpoints in the 2.6 family'; 'Both base and post-training checkpoints of the 2.6 family are open-sourced'" } }, "scale": { "total_params": { "value": 1000, "provenance": "stated", "ref": "Abstract ('This model family scales from 104B to 1T parameters'); model name Ling-2.6-1T", "note": "report never prints an exact parameter count; 1T is the disclosed scale of the reference variant" }, "active_params_per_token": { "value": null, "provenance": "unknown", "note": "no active-parameter count disclosed; only the routing shape (8 of 256 routed experts + 1 shared per layer) is given" }, "layers": { "value": 80, "provenance": "stated", "ref": "§2.1.1 ('the number of Transformer layers to 80'); Table 1 (# Layers 80)" }, "hidden_dim": { "value": 8192, "provenance": "stated", "ref": "§2.1.1 ('the hidden dimension d to 8,192'); Table 1 (Hidden Size 8,192)" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "ref": "§2.1.1 ('the intermediate hidden dimension of each routed expert is 2,048'); Table 1 (Expert Intermediate Size 2,048)", "note": "per-expert MoE width; the 4 dense FFN blocks use intermediate size 18,432 (Table 1, Intermediate Size)" }, "ffn_ratio": { "value": 0.25, "provenance": "derived", "note": "2048 / 8192 = 0.25 (per-expert ratio for the MoE FFNs; dense blocks use 18,432 / 8,192 = 2.25)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "§2.1.1 ('64 attention heads'); Table 1 (# Attention Heads 64)" }, "attention_heads_kv": { "value": 64, "provenance": "derived", "note": "Lightning Attention layers are standard MHA after conversion (§2.1.3: 'the GQA dimensions are expanded to standard Multi-Head Attention (MHA) by augmenting the Wqkv projection along the head dimension'), so KV heads = Q heads = 64 there; MLA layers cache a low-rank latent KV (KV LoRA Rank 512, Table 1) and have no per-head KV cache" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "§2.1.1 ('64 attention heads with a head dimension of 128'); consistent with 8,192 / 64" }, "vocab_size": { "value": 157184, "provenance": "stated", "ref": "§2.1.1 ('a vocabulary size of 157,184')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "input/output embedding tying not disclosed" }, "variants": [ { "name": { "value": "Ling-2.6-flash", "provenance": "stated", "ref": "Abstract; Table 1" }, "total_params": { "value": 104, "provenance": "inferred", "note": "family 'scales from 104B to 1T parameters' with three named weights; flash is the only sub-1T member (Abstract)" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 1 (Hidden Size 4,096)" }, "ffn_inner_dim": { "value": 1024, "provenance": "stated", "ref": "Table 1 (Expert Intermediate Size 1,024)", "note": "per-expert MoE width; its single dense layer uses Intermediate Size 9,216" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 1 (# Attention Heads 32)" }, "attention_heads_kv": { "value": 32, "provenance": "derived", "note": "same MHA-expansion argument as the 1T model (§2.1.3); MLA layers use latent KV (KV LoRA Rank 512, Table 1)" }, "context_length": { "value": null, "provenance": "unknown", "note": "max context length for flash not stated; the 262,144-token figure is given only for the 1T config (§2.1.1)" } }, { "name": { "value": "Ling-2.6-1T", "provenance": "stated", "ref": "Abstract; Table 1" }, "total_params": { "value": 1000, "provenance": "stated", "ref": "Abstract ('scales from 104B to 1T parameters')", "note": "exact count never printed" }, "layers": { "value": 80, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": 8192, "provenance": "stated", "ref": "Table 1" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "ref": "Table 1 (Expert Intermediate Size 2,048)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 64, "provenance": "derived", "note": "MHA on Lightning Attention layers (§2.1.3); MLA layers use latent KV" }, "context_length": { "value": 262144, "provenance": "stated", "ref": "§2.1.1 ('supports a maximum context length of 262,144 tokens')" } }, { "name": { "value": "Ring-2.6-1T", "provenance": "stated", "ref": "Abstract; §3.2" }, "total_params": { "value": 1000, "provenance": "inferred", "note": "Ring-2.6 post-training starts from the shared Ling-2.6-1T base (Figure 6: 'Ling-2.6-1T Base' -> Ring-2.6-1T); no separate config printed" }, "layers": { "value": 80, "provenance": "inferred", "note": "shares the Ling-2.6-1T base (§3, Figure 6); Table 1 lists only flash and 1T columns" }, "hidden_dim": { "value": 8192, "provenance": "inferred", "note": "shares the Ling-2.6-1T base" }, "ffn_inner_dim": { "value": 2048, "provenance": "inferred", "note": "shares the Ling-2.6-1T base" }, "attention_heads_q": { "value": 64, "provenance": "inferred", "note": "shares the Ling-2.6-1T base" }, "attention_heads_kv": { "value": 64, "provenance": "inferred", "note": "shares the Ling-2.6-1T base (MHA linear layers; MLA latent KV)" }, "context_length": { "value": 262144, "provenance": "inferred", "note": "shares the Ling-2.6-1T base; maximum conversation length for agentic RL is 200 training / 500 evaluation turns (§3.2.2)" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "76/80 = 95% of layers use routed-MoE FFNs; only the first 4 Transformer blocks are dense (Table 1 '# Dense Layers 4', §2.1.1)" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "§2.1.1 ('256 routed experts'); Table 1 (# Experts (total) 256)" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§2.1.1 ('8 experts are activated for each token'); Table 1 (# Experts Active per Token 8)" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "§2.1.1 ('1 shared expert'); Table 1 (# Shared Experts 1)" }, "routing": { "value": "grouped routing: ngroup = 8 groups with top-4 group selection; router scores with sigmoid in FP32; routed output scaled by 2.5 with normalized top-k probabilities", "provenance": "stated", "ref": "§2.1.1" }, "load_balancing": { "value": "auxiliary-loss-free load balancing with expert bias (bias-update rate gamma = 0.001 during continue pre-training, reduced to 0.0001 for mid-training)", "provenance": "stated", "ref": "§2.1.1, §2.3.1" }, "expert_granularity": { "value": "fine-grained MoE inherited from Ling-2.0: 1 shared + 256 routed experts per layer, per-expert intermediate width 2,048 (1,024 for flash)", "provenance": "stated", "ref": "§2.1.1 ('Following Ling-2.0, Ling-2.6-1T-base employs a fine-grained MoE architecture'); Table 1" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.1 ('a hybrid linear attention design that integrates Lightning Attention with MLA at a 7:1 ratio')" }, "attention_layer_pattern": { "value": "layer groups of M = 8 throughout the 80-layer stack: 7 Lightning Attention (linear) layers + 1 Full Attention (MLA) layer per group, i.e. 70 linear + 10 MLA layers total; the in-group position of the MLA layer is not disclosed", "provenance": "stated", "ref": "§2.1.2 ('each group contains 1 Full Attention (MLA) layer and M-1 Linear Attention layers'; M = 8 best); Table 1 (Layer Group Size 8)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no learned attention over prior layer outputs or connection matrix described; standard residual stacking with RMSNorm" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1.1 ('supports a maximum context length of 262,144 tokens via RoPE (Su et al., 2024) positional encoding with theta = 6,000,000')" }, "rope_base": { "value": 6000000, "provenance": "stated", "ref": "§2.1.1" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§2.1.1 ('The rotary dimension is set to 64, i.e., Partial RoPE is applied to a subset of head dimensions.')" }, "extension": { "method": { "value": "none", "provenance": "stated", "note": "training ran at the full deployed length: mid-training phase 3 covers 525B tokens at 256K context (§2.3.2) and post-training SFT corpus is at 256K (§3.1.1); the 4K -> 32K -> 256K ramp is a training curriculum, not a post-hoc extension", "ref": "§2.3.2, §3.1.1" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.3.2 (no rescaling factor; training at deployed length)" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§2.1.1" } } }, "normalization": { "type": { "value": "RMSNorm (epsilon = 1e-6)", "provenance": "stated", "ref": "§2.1.1 ('RMSNorm with epsilon = 10-6 for layer normalization'); group RMSNorm + sigmoid-gate fusion appears in the inference kernels (§4.3)" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": false, "provenance": "stated", "ref": "§2.1.3", "note": "QK Norm existed on the original Ling-2.0 GQA layers and was retained on the Lightning Attention layers during conversion, but was removed before the MLA conversion by calibration-based absorption into Wq/Wk ('QK Norm is a nonlinear operation that prevents the KV weight matrix absorption required for efficient MLA inference. We resolve this by removing QK Norm prior to the MLA conversion.') — the global-attention (MLA) layers carry no QK-norm in the final model" } }, "activation": { "value": "SiLU", "provenance": "stated", "ref": "§2.1.1 ('We employ SiLU as the activation function'); §4.3 calls the fused gated FFN 'SwiGLU' (fc1 -> SiLU gate, multiply with fc2)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no logit softcapping described" }, "other": [ { "value": "QK Norm retained on Lightning Attention layers during conversion 'to stabilize training, improve FP8 training compatibility, and enhance robustness at long-context windows'", "provenance": "stated", "ref": "§2.1.3" }, { "value": "FP32 precision for the MoE router (sigmoid scoring) and FP32 LM Head in both training and inference engines to control training-inference drift", "provenance": "stated", "ref": "§2.1.1, §4.2" }, { "value": "Module-aware FP8 quantization: attention linears and shared-expert linears stay in BF16, only routed-expert linears use blockwise FP8", "provenance": "stated", "ref": "§4.2" }, { "value": "int64 token indexing/counting in MoE kernels to survive expert token counts beyond 32-bit range in long-context training", "provenance": "stated", "ref": "§4.1.3" }, { "value": "Phase-specific GPU memory policies: aggressive in pre-training, conservative in long-context post-training, to absorb router-imbalance volatility", "provenance": "stated", "ref": "§4.1.3" } ] } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§2.1.1 ('maximum context length of 262,144 tokens'); §2.3.2 (mid-training extends 4K -> 32K -> 256K); §3.1.1 (post-training at 256K)" }, "deployed_length": { "value": 262144, "provenance": "stated", "ref": "§2.1.1" }, "extension_method": { "value": "none — training ran at the full deployed length via a progressive context curriculum (4K -> 32K -> 256K during pre/mid-training; post-training SFT at 256K); no post-hoc extension method used", "provenance": "stated", "ref": "§2.3.2, §3.1.1" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer algorithm never described; only the 157,184 vocab size appears (§2.1.1)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 9.6, "provenance": "stated", "ref": "§2.3 ('processes approximately 9.6T tokens across three stages'); Abstract ('continue pre-training ... for about 9.6T tokens')", "note": "composed of ~400B migration pre-training + 8T continue pre-training + ~1.2T mid-training (§2.3.2)" }, "data_composition": { "value": "Three-stage mixture. Migration/continue pre-training: ~46% reasoning-intensive domains (math, code), ~50% general corpora (web text), ~4% multilingual; agentic corpus spanning tool use (500+ real-world MCP environments, 3,000+ tools) and agentic coding with bash/web/repository tasks, teacher-generated with verification; long-context corpus (math, web parsing, summarization, RAG fusion, multi-hop reasoning) with rule+model defect detection; web corpus with STEM recall and atomic-fact construction from Wikipedia; multilingual coverage for 21 languages including 1.1T tokens from Fineweb2/Fineweb2-hq and ~70B synthetic web code. Mid-training strips general web data to ~43% general / 42% reasoning / 15% agentic at 256K.", "provenance": "stated", "ref": "§2.2, §2.3.2" }, "curriculum": { "value": "Multi-stage: (1) Migration Pre-Training (~400B tokens, four steps: Lightning Attention Conversion -> Linear Warmup -> MLA Conversion [QK Norm absorption, partial-parameter training, Partial-RoPE-aware TransMLA conversion] -> MLA Warmup); (2) Continue Pre-Training (8T tokens, 4K context, aggressive data-switching strategy adopted over conservative); (3) Mid-Training (~1.2T tokens: 250B at mixed 32K, then 425B at 32K, then 525B at 256K, high-quality mixture). WSM scheduler throughout.", "provenance": "stated", "ref": "§2.3.2" }, "optimizer": { "value": null, "provenance": "unknown", "note": "optimizer not named; §2.3.1 defers ('All other hyper-parameters remain consistent with Ling-2.0'); Muon is mentioned only as a future direction (§5)" }, "lr_schedule": { "value": "WSM scheduler from Ling-2.0: linear warmup to peak learning rate, constant phase until training concludes, final annealing achieved through checkpoint merging", "provenance": "stated", "ref": "§2.3.1" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "batch size values not printed; scaling laws were re-established on the hybrid architecture to pick learning rate and batch size (§2.3.1)" }, "precision": { "value": "FP8 training from pretraining through supervised fine-tuning, with FP32 optimizer master weights retained for RL initialization; FP8 continuation training for RL; FP32 LM Head in both training and inference", "provenance": "stated", "ref": "§4.2" }, "parallelism": { "value": "co-designed expert, pipeline, and context parallelism with selective activation recomputation; AllGather-based Context Parallel for Lightning Attention (local-recurrence-then-global-correction, no head-divisibility constraint); Triton-fused varlen state-correction kernel (~68% end-to-end speedup at 256K)", "provenance": "stated", "ref": "§4.1" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed (H20 appears only in the Ling-2.6-flash serving benchmark, §3.3.1)" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 ('an initial cold-start SFT phase followed by specialized expert fine-tuning'); §3.2 (Ring-2.6 cold-start SFT)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§3.1.3", "note": "bidirectional preference alignment: a single reward model combining positive incentives and negative penalties, with a focus-reward mechanism that shifts training weights away from saturated rubric dimensions; ReportLogic evaluator for long-form writing; agentic tasks use GSPO (§3.1.2). No RLHF/DPO/GRPO named." }, "reasoning_training": { "value": "Evo-CoT RL (from Ling-2.0) with composite rewards: accuracy, formatting penalty for reasoning markers, dynamic length penalty (difficulty-specific), and LLM-judge semantic redundancy penalty; Linguistic Unit Policy Optimization (LPO); Dynamic Pass Rating (DPR) adaptive curriculum; GSPO for token-efficient tool use (process reward on tool-call sequence + zlib-compression repetition penalty); Ring-2.6 adds KPop agentic RL (binary-KL divergence masking) and adaptive thinking with high/xhigh effort modes", "provenance": "stated", "ref": "§3.1.2, §3.1.3, §3.2.2, §3.2.3" }, "distillation": { "value": "specialist distillation: reasoning and agentic specialist models are distilled back into the unified Ling-2.6 / Ring-2.6 model (§3.1, §3.2); shortest-correct-response distillation, where proprietary expert models generate SFT responses and only the shortest accurate candidate is retained (§3.1.2)", "provenance": "stated", "ref": "§3.1, §3.2" } }, "modality": { "type": { "value": "text-only", "provenance": "stated", "ref": "§5 ('an equally important transition is from text-only systems to native multimodal agents')" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§5 (model is text-only)" } }, "efficiency": { "kv_cache": { "value": "hybrid: MLA layers compress the KV cache into a low-rank latent space (KV LoRA Rank 512, Q LoRA Rank 1536, Table 1); Lightning Attention layers use an O(N) linear-attention recurrence with no growing KV cache. The 7:1 mix 'reduces long-context compute cost, KV-cache pressure, and decoding latency'.", "provenance": "stated", "ref": "§2.1, Table 1" }, "quantization": { "value": "FP8 training and inference; module-aware FP8 quantization at deployment (attention linears and shared-expert linears in BF16, routed-expert linears blockwise FP8); BF16 inference path also supported; BF16-input/FP32-output router and LM-Head GEMMs", "provenance": "stated", "ref": "§4.2, §4.3" }, "speculative_dedup": { "value": "MTP: two additional MTP layers added during post-training and continued-trained; MTP layers serve as a draft model for speculative decoding (4 speculative steps); parameter sharing across MTP layers with gradient detachment from all but the first (MTP-3-share) raises accepted length to 3.31 vs 2.71 for MTP-1", "provenance": "stated", "ref": "§3.1, Appendix B" }, "serving": { "value": "linghe fused-kernel library (open-sourced): QK Norm + RoPE fusion, group RMSNorm + sigmoid-gate fusion, MLA RoPE and Top-K optimization, split-K blockwise FP8 GEMM for small batches; prefix caching and multi-token generation; AllGather context parallelism; ARouter global rollout scheduling with tail-request migration, spillover training-inference overlap, and instance failover; up to 4x prefill/decode acceleration for Ling-2.6-flash vs similar-size peers", "provenance": "stated", "ref": "§3.3.1, §4.1.1, §4.2, §4.3" } }, "evaluation": { "benchmarks": [ { "name": { "value": "Artificial Analysis Intelligence Index", "provenance": "stated", "ref": "Abstract, §3.3.1" }, "value": { "value": 34, "provenance": "stated", "ref": "Abstract ('a score of 34 using only about 16M output tokens')" }, "ref": { "value": "Abstract, §3.3.1", "provenance": "stated", "ref": "Abstract, §3.3.1" } }, { "name": { "value": "C-SimpleQA (Correct)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 76.53, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SimpleQA-Verified (Correct)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 31.5, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "GPQA-Diamond (Mean@4, CoT)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 76.17, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "AIME26 (Mean@64, CoT)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 87.4, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HMMT-Nov25 (Mean@64, CoT)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 81.93, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "IMO-AnswerBench (Mean@8, CoT)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 65.81, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LiveCodeBench-v6 (Mean@4)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 65.58, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "bbeh (Pass@1)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 52.37, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "ARCPrize (Mean@4)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 50.94, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SWE-bench Verified (Claude Code, openhands fc)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 72.2, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "PinchBench (Mean@5)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 85.24, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "ClawEval (Pass@3)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 51.0, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BFCL-v4 (Accuracy)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 70.64, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "tau2-bench (Mean@4)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 78.36, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "terminal-bench 2.0 (Accuracy)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 40.45, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "IFBench (Mean@5)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 57.62, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MRCR (16K-256K, Mean)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 80.37, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LongBenchV2 (Accuracy)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 48.31, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "AIME 2026 (Avg@64) — Ring-2.6-1T xhigh", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 95.78, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "LCB-v6 (Avg@4) — Ring-2.6-1T xhigh", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 86.95, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GPQA-Diamond (Avg@16) — Ring-2.6-1T xhigh", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 85.89, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "ARC-AGI-2 (Pass@2) — Ring-2.6-1T xhigh", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 66.18, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "PinchBench (Avg@3) — Ring-2.6-1T high", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 87.6, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "ClawEval (0424, Pass^3) — Ring-2.6-1T high", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 63.82, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SWE-bench Verified (Resolved) — Ring-2.6-1T high", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 74.0, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SWE-bench Pro (Resolved) — Ring-2.6-1T high", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 53.76, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GAIA-2 Search (Pass@1, 3 runs) — Ring-2.6-1T xhigh", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 77.9, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "tau2-Average (Acc) — Ring-2.6-1T high", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 84.26, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMLU (EM) — Ling-2.6-1T-base", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 86.82, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LongBenchv2 (Acc) — Ling-2.6-1T-base", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 43.54, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Hybrid linear attention retrofit: Lightning Attention + MLA at a 7:1 ratio chosen by equal-FLOPs scaling laws, transplanted into the trained Ling-2.0-1T checkpoint via a four-step lossless migration (hybrid initialization, QK Norm absorption, Partial-RoPE-aware TransMLA conversion, MLA warmup)", "provenance": "stated", "ref": "Abstract, §2.1" }, "ref": { "value": "Abstract, §2.1", "provenance": "stated", "ref": "Abstract, §2.1" } }, { "text": { "value": "Token-efficiency post-training suite — Evo-CoT, Linguistic Unit Policy Optimization (LPO), bidirectional preference alignment, and shortest-correct-response distillation — delivering approximately 4x higher token efficiency on reasoning workloads than the 2.0 generation (AA Intelligence Index 34 at ~16M output tokens)", "provenance": "stated", "ref": "Abstract, §3.1" }, "ref": { "value": "Abstract, §3.1", "provenance": "stated", "ref": "Abstract, §3.1" } }, { "text": { "value": "KPop, a novel RL algorithm that replaces IcePop's uniform fixed-ratio constraint with symmetric binary KL divergence masking, enabling stable trillion-parameter-scale agentic RL (SWE-bench Verified solve rate 70.8% -> 76.28% during training)", "provenance": "stated", "ref": "§3.2.3" }, "ref": { "value": "§3.2.3", "provenance": "stated", "ref": "§3.2.3" } }, { "text": { "value": "Native agentic optimization: agentic pre-training corpus (500+ real-world MCP environments, 3,000+ tools), reproducible Dockerized coding environments (AEnvironment, ~220,000 images), and Dynamic Pass Rating (DPR) adaptive curriculum", "provenance": "stated", "ref": "§2.2.1, §3.2.1, Appendix A" }, "ref": { "value": "§2.2.1, §3.2.1, Appendix A", "provenance": "stated", "ref": "§2.2.1, §3.2.1, Appendix A" } }, { "text": { "value": "Infrastructure co-design: AllGather context parallelism for linear attention (free of head-divisibility constraints), linghe fused-kernel library, continued MTP training for speculative decoding, ARouter asynchronous RL with bounded staleness, and integrated FP8 training/inference", "provenance": "stated", "ref": "§4" }, "ref": { "value": "§4", "provenance": "stated", "ref": "§4" } } ], "assessment": { "value": "The closest atlas peer is moonshot-kimi-linear-48b-a3b: both are text-only, open-weights hybrids of a linear-attention family with MLA over a fine-grained MoE backbone aimed at agentic/long-context efficiency — and Kimi Linear's own hybrid-ratio ablation explicitly tested and rejected 7:1 in favor of 3:1, which Ling-2.6 adopts on its own scaling-law evidence. The genuinely distinctive axis is that Ling-2.6 is not trained from scratch: it is an architectural transplant of a trained trillion-parameter GQA checkpoint (the QK-Norm absorption via calibration statistics and the Partial-RoPE-decoupled TransMLA conversion are a novel retrofit procedure with no atlas precedent), and it uses Lightning Attention (TransNormerLLM lineage) rather than the delta-rule KDA used by both Moonshot records. The token-efficiency post-training direction (Evo-CoT, LPO, bidirectional preference alignment) and the grouped-routing fine-grained MoE inherited from Ling-2.0 are family-specific rather than novel-at-large; KPop is an incremental refinement of the team's own IcePop (Ring-1T), which is not itself an atlas record. The '1T' scale, ~4x token-efficiency gain, and 87.6 PinchBench lead are the report's own figures and not independently verifiable from the atlas.", "provenance": "inferred", "note": "authored by atlas analyst; comparison against atlas entries (moonshot-kimi-linear-48b-a3b, moonshot-kimi-k3, deepseek-v3, jamba) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "76/80 = 95% of layers use routed-MoE FFNs (first 4 blocks dense, Table 1) — exceeds the 80% majority threshold" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "70/80 = 87.5% of attention layers are Lightning Attention (linear-attention family), 10/80 MLA — >= 80% majority threshold for efficient-attention; deliberate per-group 7:1 pattern (M = 8)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (§5)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (all checkpoints open-sourced); data composition described in detail but corpora not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "1000B total params >= 100B (reference variant Ling-2.6-1T)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as 'practical agentic intelligence at trillion-parameter scale' pursuing three simultaneous goals: long-context efficiency, capability per output token, and native agentic optimization; the family splits into an instant-response line (Ling-2.6) and a deep-reasoning line (Ring-2.6), with token efficiency and agentic behavior as secondary emphases noted in prose" } }, "prose": { "design_overview": { "value": "Ling-2.6-1T is an 80-layer, ~1T-parameter sparse MoE (256 routed experts, 8 active, 1 shared per layer; 4 dense blocks up front) that is remarkable less for its block than for how the block was obtained: it is an architectural transplant of the trained Ling-2.0-1T GQA checkpoint rather than a from-scratch model. A four-step migration (Lightning Attention Conversion, Linear Warmup, MLA Conversion, MLA Warmup) replaced most GQA layers with Lightning Attention — a simple O(N) linear attention — and converted the remaining full-attention layers from GQA to MLA, fusing the old QK Norm into the projections by calibration and adapting the Partial RoPE (rotary dim 64, theta 6M) to TransMLA. The final block mixes 7 linear-attention layers with 1 MLA layer per group of 8 (70:10 over the stack), keeping KV-cache pressure and long-context FLOPs low at the 262,144-token context, which is reached by a 4K-to-256K training curriculum rather than any post-hoc extension. FFNs are fine-grained MoE with grouped routing (8 groups, top-4), sigmoid FP32 scoring, expert bias for auxiliary-loss-free balancing, and a 2.5x output scale. Post-training diverges into two lines from the shared base: Ling-2.6 is an instant model whose Evo-CoT/LPO RL, bidirectional preference alignment, and shortest-correct-response distillation push capability per output token (~4x token efficiency claimed), while Ring-2.6 adds KPop agentic RL (binary-KL masking) and adaptive thinking for long-horizon tool use. Training is FP8 throughout, served by the linghe fused kernels and an AllGather context-parallel scheme, with MTP layers doubling as speculative-decoding drafts.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Lightning Attention — linear attention with O(N) recurrence used as the majority mixing layer", "provenance": "stated", "ref": "§2.1 (cites Qin et al., 2024, TransNormerLLM)" }, "origin": { "value": "TransNormerLLM (not in atlas)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Multi-head Latent Attention (MLA) — low-rank latent KV compression for the full-attention layers", "provenance": "stated", "ref": "§2.1 (cites DeepSeek-AI et al., 2024, DeepSeek-V2)" }, "origin": { "value": "DeepSeek-V2", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "TransMLA weight-conversion method for GQA-to-MLA structural conversion, adapted to Partial RoPE by decoupling the RoPE-affected dimensions", "provenance": "stated", "ref": "§2.1.3 (cites Meng et al., 2025)" }, "origin": { "value": "TransMLA (not in atlas)", "provenance": "stated", "ref": "§2.1.3" } }, { "mechanism": { "value": "Hybrid linear-attention conversion recipe (GQA dims expanded to MHA, gating parameters Wgate / gamma_gate introduced)", "provenance": "stated", "ref": "§2.1.3 ('following the Ring-flash-linear-2.0', Team et al., 2025a, own prior work)" }, "origin": { "value": "Ring-flash-linear-2.0 (Inclusion AI, own prior work; not in atlas)", "provenance": "stated", "ref": "§2.1.3" } }, { "mechanism": { "value": "GQA backbone, fine-grained MoE with grouped routing, Evo-CoT framework, and WSM learning-rate scheduler", "provenance": "stated", "ref": "§2.1.1, §2.3.1, §3.1.2 (Team et al., 2025b, Ling-2.0)" }, "origin": { "value": "Ling-2.0 (Inclusion AI, own prior work; not in atlas)", "provenance": "stated", "ref": "§2.1.1, §2.3.1" } }, { "mechanism": { "value": "IcePop double-sided-masking RL stabilization — KPop replaces its uniform constant-ratio constraint with binary KL divergence", "provenance": "stated", "ref": "§3.2.3 (Ling Team, 2025, Ring-1T)" }, "origin": { "value": "Ring-1T (Inclusion AI, own prior work; not in atlas)", "provenance": "stated", "ref": "§3.2.3" } }, { "mechanism": { "value": "Group Sequence Policy Optimization (GSPO) for token-efficient agentic RL", "provenance": "stated", "ref": "§3.1.2 (cites Zheng et al., 2025)" }, "origin": { "value": "GSPO (Zheng et al., 2025; not in atlas)", "provenance": "stated", "ref": "§3.1.2" } } ], "influence_out": [], "notable_omissions": { "value": [ "Exact parameter counts never printed — only 'scales from 104B to 1T parameters'; no active-parameters-per-token figure (the 8/256 routing shape is given but not the resulting active count)", "Training hardware and compute (FLOPs/GPU-hours) not disclosed", "Optimizer not named ('all other hyper-parameters remain consistent with Ling-2.0'); Muon appears only as a future direction", "Peak learning rate and batch size values not printed (scaling laws re-derived but figures withheld)", "Tokenizer algorithm not described (only the 157,184 vocab size appears)", "Input/output embedding tying not disclosed", "Normalization placement (pre/post) not described", "No license named for the open-sourced checkpoints", "Ring-2.6-1T's architecture is not in Table 1 (only flash and 1T columns); its identity as the shared 1T base is inferred from Figure 6", "The position of the single MLA layer within each 8-layer group is not disclosed", "KPop technical details deferred to an external blog (ringtech.notion.site/kpop)", "Some Table 6 peer numbers are cited from other labs or marked 'results from our evaluation' (*), and several cells are missing (-); SWE benchmarks use Claude Code scaffolding rather than the model's own agent harness", "No attention-sink or logit-softcapping mechanisms discussed (absence defaults apply)", "Source extraction: ar5iv HTML conversion failed (fatal conversion banner, 229 chars), so text was extracted from the official arXiv PDF (fitz); embedded formulas and figure-internal labels are partially garbled (e.g., Eq. 5, Table 6 footnote markers), and a handful of control bytes were stripped during cleaning" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "inclusion-ling-2-6-1t.json" }, { "slug": { "value": "jetbrains-mellum2-thinking-12b-a2-5b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2605.31268", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/jetbrains-mellum2-thinking-12b-a2-5b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Mellum 2", "provenance": "stated", "ref": "Title ('Mellum 2 Technical Report'), Abstract" }, "variants": [], "org": { "value": "JetBrains (with Constructor University, Bremen)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-05-29", "provenance": "stated", "ref": "arXiv header line 16 ('arXiv:2605.31268v1 [cs.CL] 29 May 2026'); title page stamps 'v1.0 · May 2026'" }, "report_url": { "value": "https://arxiv.org/abs/2605.31268", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "Abstract ('released… under the Apache 2.0 license'); §1" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('We release the base, instruct, and thinking checkpoints'); §1 ('We release base, instruct, and thinking checkpoints under the Apache 2.0 license, together with this report'; 'we release a base model before the long context extension and SFT checkpoints')" } }, "scale": { "total_params": { "value": 12, "provenance": "stated", "ref": "Abstract, §2.2, Table 2 ('≈12B total parameters' / '≈12B')" }, "active_params_per_token": { "value": 2.5, "provenance": "stated", "ref": "Abstract, §2.2, Table 2 ('≈2.5B active parameters per token')" }, "layers": { "value": 28, "provenance": "stated", "ref": "§2.2 ('Backbone: 28 transformer layers'), Table 2" }, "hidden_dim": { "value": 2304, "provenance": "stated", "ref": "§2.2 ('hidden dimension 2,304'), Table 2" }, "ffn_inner_dim": { "value": 896, "provenance": "stated", "ref": "§2.2 ('expert intermediate size 896'), Table 2 ('Expert MLP size 896') — per-expert inner dim (MoE)" }, "ffn_ratio": { "value": 0.39, "provenance": "derived", "note": "896 / 2304 = 0.3889 (per-expert inner dim over hidden dim, MoE)" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "§2.2 ('32 query heads'), Table 2" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "§2.2 ('4 KV heads (GQA)'), Table 2" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "§2.2 ('head dimension 128'), Table 2" }, "vocab_size": { "value": 98304, "provenance": "stated", "ref": "§2.2 ('a 98,304-token vocabulary'), Table 2, §3.1.4" }, "embedding_tied": { "value": false, "provenance": "stated", "ref": "§2.2 ('untied input/output embeddings'), Table 2 ('Tied embeddings: No')" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "all 28 layers carry an MoE FFN ('all FFN layers in Mellum 2 are MoE', Appendix A.2); the report calls it a MoE model" }, "moe": { "expert_count": { "value": 64, "provenance": "stated", "ref": "§2.2 ('64 routed experts per layer'), Table 2" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§2.2 ('8 active per token (top-8 routing)'), Table 2" }, "shared_experts": { "value": 0, "provenance": "stated", "ref": "§2.2 ('no shared expert'), Table 2 ('Shared expert: None'); Appendix A.2 (a shared expert was ablated and dropped: 'no measurable quality gain… consistently hurt inference performance')" }, "routing": { "value": "top-8 expert selection; router operates in FP32; dropless routing (no expert capacity factor, no token dropping)", "provenance": "stated", "ref": "§2.2, §3.4.5, Table 11" }, "load_balancing": { "value": "global-batch auxiliary load-balancing loss (coefficient 1e-3) plus router z-loss (coefficient 1e-3); global-batch balancing chosen over per-sequence balancing for flexibility (per-sequence gave marginally better loss on short runs); aux-loss coefficient reduced to 1e-4 during SFT; auxiliary-loss-free bias balancing (DeepSeek-V3 style) considered but rejected for ecosystem fit", "provenance": "stated", "ref": "§3.4.5, Appendix A.2, §5.1.2, Table 11" }, "expert_granularity": { "value": "fine-grained: 64 experts per layer with 8 active (12.5% activation ratio), expert intermediate size 896; scaled down proportionally from the Qwen3-30B-A3B architecture", "provenance": "inferred", "note": "evidence: §2.2 and §2.1.2 (fixed at 64 experts as the maximum fitting GPU memory; 8-of-64 sparsity chosen as the quality-latency optimum); the report itself does not use the term 'fine-grained'" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.1.4, §2.2 (GQA with a 3:1 Sliding Window Attention pattern: 3 of every 4 layers use a 1,024-token sliding window, the remaining layer uses full attention; 32 query heads / 4 KV heads, QK-Norm, RoPE)", "note": "all layers are GQA-based; the hybrid is between full-attention GQA layers and sliding-window GQA layers (75% SWA / 25% full — no 80% majority, so not classified as pure sliding-window)" }, "attention_layer_pattern": { "value": "3:1 SWA: 3 out of every 4 layers use a sliding window of 1,024 tokens and the remaining 1 in 4 uses full attention", "provenance": "stated", "ref": "§2.1.4, §2.2, Table 2" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no learned attention over prior layer outputs described; standard pre-RMSNorm residual stack" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.2 ('Rotary Position Embeddings (RoPE) with base θ=500,000'), Table 2" }, "rope_base": { "value": 500000, "provenance": "stated", "ref": "§2.2 ('RoPE with base θ=500,000'), Table 2" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "RoPE dimension fraction not stated" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "ref": "§4.1 (layer-selective YaRN: frequency re-mapping applied only to the global (full-attention) layers; sliding-window layers keep their original RoPE parameters — recipe first reported in Gemma 3 and adopted by OLMo 3)" }, "factor": { "value": null, "provenance": "unknown", "note": "YaRN scaling factor not stated in the report" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§4 ('extend the effective context length of Mellum 2 from the 8,192-token training context to 131,072 tokens (128K)'), Table 2" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.2 ('pre-RMSNorm (ε=10^-6)'), Table 2; QK-Norm is also RMSNorm (Table 2: 'QK-Norm: Yes (RMSNorm)')" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§2.2 ('pre-RMSNorm')" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2.2 ('QK-Norm applied to the query and key projections'), Table 2 ('QK-Norm: Yes (RMSNorm)')" } }, "activation": { "value": "SiLU-gated MLP (SwiGLU)", "provenance": "stated", "ref": "§2.2 ('SiLU-gated MLPs'), Table 2 ('Activation: SiLU (gated)')" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink mechanism described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no softcapping described" }, "other": [ { "value": "Router z-loss (ST-MoE, coefficient 1e-3) for MoE training stability", "provenance": "stated", "ref": "§3.4.5, Table 11" }, { "value": "FP32 gradient reduction under BF16 + FP8 hybrid precision for numerical stability", "provenance": "stated", "ref": "§3.4.4, Table 11" }, { "value": "IcePop per-token truncation of train-versus-inference importance ratios in RL: tokens whose ratio falls outside [0.5, 5.0] are dropped, guarding against MoE expert-flip divergence between the inference router and the trainer-side router", "provenance": "stated", "ref": "§5.2.3" }, { "value": "ARLCP-style concision penalty in the Instruct RL run: multiplicative reward shrink on correct rollouts proportional to reflection-trigger-word count, suppressing inline reasoning leakage in the no-thinking variant (7.3 → 0.6 trigger words per rollout)", "provenance": "stated", "ref": "§5.2.3 (Reward shaping)" }, { "value": "Data-side stability fixes: filtering samples with <82 unique tokens (1% of context) to remove low-diversity loss spikes; hash-sorted duplicate chunks caused 16 periodic loss dips per phase and were accepted as benign", "provenance": "stated", "ref": "§3.6" } ] } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§4 (native 8,192-token pre-training context extended to 131,072 in the long-context stage; SFT and RL train on packed 131,072-token sequences — §5.1.2, Table 6)" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§4 ('to 131,072 tokens (128K)'); Abstract ('extended to a 128K context window'); Table 2 ('8,192 / 131,072⋆ after the long-context extension stage')" }, "extension_method": { "value": "layer-selective YaRN: frequency re-mapping applied only to global (full-attention) layers, sliding-window layers keep original RoPE (Gemma 3 / OLMo 3 recipe); ~117B tokens of long-context training (3,500 iterations, WHD schedule, peak LR 3e-5, 500 decay iterations); RULER @64K 0.64 vs 0.52 uniform θ-bump vs 0.33 unchanged-θ", "provenance": "stated", "ref": "§4.1, §4.3" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "a custom tokenizer is used but its algorithm (BPE/SentencePiece/etc.) is never named" }, "notes": { "value": "Custom tokenizer with a 98,304-token vocabulary, identical to the tokenizer used in Mellum-4B; designed to provide strong coverage of programming-language tokens and technical terminology", "provenance": "stated", "ref": "§3.1.4" } }, "training": { "tokens": { "value": 10.65, "provenance": "stated", "ref": "§3 ('approximately 10.6 trillion tokens'); Table 3 (phase totals 6.18 + 2.79 + 1.69 = 10.65T)" }, "data_composition": { "value": "Three categories: (1) web and general knowledge — large-scale synthetic web corpora from Common Crawl, educational web content and PDFs, multilingual reasoning/QA datasets, curated knowledge (SFT data, STEM instruction data, Wikipedia rewrites, synthetic encyclopedic articles); (2) source code — raw permissively licensed code deduplicated at file level, code extracted from Common Crawl, and synthetic/derived datasets (summarization, functionality extensions, translations, test generation, commit messages, task descriptions, QA, rewriting, review, transpilation, educational explanations); (3) mathematical data — math SFT data, quality-tiered math web content, permissively licensed textbooks, math instruction data; raw code corpus seen for 3 epochs (~958B tokens), no dataset repeated more than 4×", "provenance": "stated", "ref": "§3.1, §3.2.1" }, "curriculum": { "value": "Three-phase curriculum aligned with the WHD schedule: Phase 1 Foundation (~6.18T tokens, 58%; web 70% / code 23% / math 6%; warmup→hold), Phase 2 Quality Uplift (~2.79T, 26.2%; web 44% / code 42% / math 14%; hold), Phase 3 Capability Sharpening (~1.69T, 15.9%; web 23% / code 59% / math 18%; decay); FIM rate schedule 50% (all data) → 10% → 50% (code files only); global batch size ramps 2,048 → 4,096 sequences", "provenance": "stated", "ref": "§3.2, Table 3" }, "optimizer": { "value": "Distributed Muon (Moonlight configuration: spectral scale mode, extra scale factor 0.2, Newton-Schulz iterations 5, momentum 0.95, Nesterov momentum, blockwise TP mode) with Adam (beta1=0.9, beta2=0.95, epsilon=1e-8) for embedding and output layers; weight decay 0.1; gradient clipping 1.0", "provenance": "stated", "ref": "§3.4.1, Table 4, Table 11" }, "lr_schedule": { "value": "Warmup-Hold-Decay (WHD): linear warmup over 2,000 steps to peak 3e-4, held through Phases 1-2, linear decay to zero over 49,306 steps (~15% of training, Phase 3); linear decay-to-zero chosen over cosine-to-nonzero minimum", "provenance": "stated", "ref": "§3.4.2, Table 4" }, "batch_schedule": { "value": "Global batch size ramps linearly from 2,048 to 4,096 sequences during the initial phase; at full batch each step processes ~33.6M tokens (4,096 × 8,192); micro-batch size 2; sequence length 8,192", "provenance": "stated", "ref": "§3.4.3, Table 4" }, "precision": { "value": "BF16 base precision with FP8 hybrid mixed precision (tensorwise FP8 recipe, most-recent amax algorithm); gradient reduction in FP32", "provenance": "stated", "ref": "§3.4.4, Table 11" }, "parallelism": { "value": "Expert parallelism 8 (each GPU hosts 8 of 64 experts), tensor parallelism 1, pipeline parallelism 1, gradient reduction and parameter gather overlapped with computation; SFT adds context parallelism 8", "provenance": "stated", "ref": "§3.4.7, Table 11, §5.1.2" }, "hardware": { "value": "32 nodes × 8 H200 GPUs (migrated to a 16-node cluster mid-training with effective batch size fixed); RL on a Kubernetes cluster of H200 nodes (small training group + larger generation group); Megatron-LM / Megatron-Bridge, NeMo-RL, vLLM, Ray", "provenance": "stated", "ref": "§3.4.7, §3.6, §5.2.1" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§5.1 (two SFT variants — Instruct and Thinking — from the same long-context YaRN checkpoint and data mix, differing in chat template, reasoning-trace handling, and loss masking; packed 131,072-token sequences, 3 epochs, peak LR 3e-5 cosine-decaying to 3e-6, Muon optimizer, MoE aux-loss coefficient reduced to 1e-4, EP=8 + CP=8 on 16×8 H200; Instruct run ≈47B tokens, Thinking run ≈167B tokens; MTP head retained with α=0.1)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§5.2.3 (GRPO variant: token-level loss, leave-one-out advantage baseline without std normalization, asymmetric clip-higher [1-0.2, 1+0.28], IcePop band [0.5, 5.0], KL coefficient 0, DAPO-style soft overlong penalty, ARLCP-style concision penalty for Instruct; RLVR rather than RLHF — every prompt has a programmatic verifier; Instruct 500 steps at max seq 16,384, Thinking 100 steps at max seq 40,960; AdamW peak 1e-6 → 1e-7)" }, "reasoning_training": { "value": "Thinking variant: emits an explicit reasoning trace before the final answer; SFT loss on the last assistant turn + reasoning field only (multi-turn conversations unfolded into up to 5 samples); RLVR with a harder thinking mix (difficulty-filtered long-form math subset); max total sequence length 40,960 tokens for thinking RL rollouts", "provenance": "stated", "ref": "§5.1, §5.2, Table 6, Table 8" }, "distillation": { "value": null, "provenance": "unknown", "note": "no model distillation described; RL data is drawn from public RLVR releases (OLMo-3 instruct/thinking math, Nemotron math-advanced-calculations and instruction-following, xLAM-style function calling, reasoning-gym, a stateful workplace-assistant benchmark) — data reuse, not teacher distillation" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: all data, objectives (next-token, FIM, MTP), and benchmarks are text/code; no vision or other modality appears" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type = text-only" } }, "efficiency": { "kv_cache": { "value": "GQA: KV cache proportional to 4 KV heads; the 3:1 SWA pattern bounds per-token KV reads on window layers to the 1,024-token window; KV-cache size under high concurrency was the dominant design driver for choosing 4 KV heads", "provenance": "derived", "note": "derived from attention_variant (§2.1.3: 'The number of KV heads is the most significant factor affecting inference throughput under high-concurrency conditions')" }, "quantization": { "value": "dynamic FP8 model quantization for serving (vLLM FP8 serving on a single H100 80GB)", "provenance": "stated", "ref": "§6" }, "speculative_dedup": { "value": "single Multi-Token Prediction (MTP) head: one additional transformer layer predicting one future token, trained with scaled loss α=0.1, removed at evaluation/inference, and used as a built-in draft model for speculative decoding", "provenance": "stated", "ref": "§2.1.5, §2.2" }, "serving": { "value": "vLLM serving on a single H100 (80GB); matches Qwen2.5-7B sync-mode latency (192 vs 193 tokens/s) and exceeds it by 21% in throughput mode (5,179 tokens/s; 20.2 req/s sustained vs 16.7 for Qwen2.5-7B and 11.3 for Qwen3-8B); workload shape ISL/OSL = 2,304/256 tokens", "provenance": "stated", "ref": "§6, Figure 13" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 70.9, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MMLU-Pro (base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 59.3, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BBH (base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 74.9, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "GSM8K (base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 81.7, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HumanEval (base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 41.5, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MBPP (base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 62.4, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LiveCodeBench v6 (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 37.2, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "EvalPlus (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 78.4, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "BFCL v3 (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 66.3, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "BFCL v4 (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 44.2, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "AIME (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 41.7, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "GSM-Plus (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 80.5, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "MMLU-Redux (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 78.1, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "GPQA Diamond (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 40.9, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "IFEval (Instruct-RL)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 75.8, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "LiveCodeBench v6 (Thinking-SFT)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 75.1, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "AIME (Thinking-RL)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 58.4, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "GSM-Plus (Thinking-RL)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 87.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "RULER @64K (layer-selective YaRN, long-context stage)", "provenance": "stated", "ref": "§4.1" }, "value": { "value": 0.64, "provenance": "stated", "ref": "§4.1" }, "ref": { "value": "§4.1", "provenance": "stated", "ref": "§4.1" } } ] }, "contributions": { "claimed": [ { "text": { "value": "An efficiency-aware architecture: every design choice (MoE vs dense, 8-of-64 expert sparsity, 4-KV-head GQA, 3:1 SWA, single MTP head) selected by ablation against a fixed inference budget — matching the single-H100 latency of Qwen2.5-7B — yielding a 12B/2.5B-active model that matches Qwen2.5-7B sync latency and exceeds it by 21% in throughput", "provenance": "stated", "ref": "§1, §2, §6, §7" }, "ref": { "value": "§1, §2, §6, §7", "provenance": "stated", "ref": "§1, §2, §6, §7" } }, { "text": { "value": "A three-phase pre-training curriculum on ~10.6T tokens ('web early, curated late'): the mixture shifts from diverse web content to curated code and math (code ratio 23% → 42% → 59%) with batch-size doubling and a capability-sharpening phase that decays the LR linearly to zero", "provenance": "stated", "ref": "§1, §3.2" }, "ref": { "value": "§1, §3.2", "provenance": "stated", "ref": "§1, §3.2" } }, { "text": { "value": "A Muon + FP8 hybrid training recipe at production scale (distributed Muon in the Moonlight configuration, tensorwise FP8 with most-recent amax, FP32 gradient reduction) with training-stability observations across the full ten-trillion-token run", "provenance": "stated", "ref": "§1, §3.4" }, "ref": { "value": "§1, §3.4", "provenance": "stated", "ref": "§1, §3.4" } }, { "text": { "value": "Long-context extension to 131,072 tokens via layer-selective YaRN — frequency re-mapping applied only to the global (full-attention) layers — with empirical findings on data-mix transfer (Longmino mix failed to replicate; ~30B tokens sufficient, 117B run for router equilibration) and MoE router dynamics during extension", "provenance": "stated", "ref": "§1, §4" }, "ref": { "value": "§1, §4", "provenance": "stated", "ref": "§1, §4" } }, { "text": { "value": "Two post-trained variants (Instruct and Thinking) from a shared long-context base via SFT followed by RLVR with a GRPO variant (IcePop truncation, clip-higher, no KL anchor, verifiable rewards only)", "provenance": "stated", "ref": "§1, §5" }, "ref": { "value": "§1, §5", "provenance": "stated", "ref": "§1, §5" } }, { "text": { "value": "Open release under Apache 2.0: base, instruct, and thinking checkpoints (plus a pre-long-context base and SFT checkpoints) with the full architecture/data/recipe documentation", "provenance": "stated", "ref": "§1, §7" }, "ref": { "value": "§1, §7", "provenance": "stated", "ref": "§1, §7" } } ], "assessment": { "value": "Architecturally derivative by design — the report openly frames Mellum 2 as the Qwen3-MoE recipe (64 experts/8 active, GQA, QK-Norm, RMSNorm, SiLU, RoPE) plus three known components: Mistral-style sliding-window attention (3:1, window 1,024), Gloeckle-style MTP, and Gemma-3/OLMo-3-style layer-selective context scaling (with YaRN instead of PI). No mechanism is new to the atlas. The genuine contributions are process-level: every choice is tied to an ablation against a fixed single-H100 latency budget (with rejected alternatives documented — MLA, dense sweeps, hybrid Gated DeltaNet backbones, shared experts, aux-loss-free balancing), the three-phase curriculum with an unusually transparent FIM schedule, and a very complete RL recipe (IcePop, clip-higher, concision penalty). vs atlas peers it sits between mistral-mixtral-8x7b (MoE, no SWA/GQA/MTP) and google-gemma-3 (SWA + layer-selective scaling but dense, 5:1 pattern); it is the first small-MoE coding-assistant record and the first Qwen3-MoE-style record (the existing qwen-qwen3-moe record predates the current schema).", "provenance": "inferred", "note": "comparison against atlas entries (google-gemma-3, mistral-mixtral-8x7b, deepseek-v3, microsoft-phi-4) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all 28 layers carry MoE FFNs ('all FFN layers in Mellum 2 are MoE', Appendix A.2)" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "3:1 SWA pattern: 75% sliding-window layers, 25% full-attention layers — below the 80% majority threshold, deliberate interleaving (contrast gemma-3's 5:1 ≈ 83% which classifies as efficient-attention)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Apache 2.0 checkpoints released); training data not released (only described)" }, "scale_class": { "value": "large", "provenance": "derived", "note": "12B (reference variant) — within the 10-100B band" }, "design_intent": { "value": "domain-specific", "provenance": "inferred", "note": "framed as a general-purpose model 'specialized in software engineering' (Abstract) — a coding assistant for in-IDE deployment (JetBrains IDEs, successor to the completion-focused Mellum-4B) with commodity-GPU inference cost as the binding design constraint; secondary: reasoning-specialised via the Thinking variant, noted in prose" } }, "prose": { "design_overview": { "value": "Mellum 2 is a 28-layer, 12B-parameter MoE (2.5B active per token) built as a scaled-down Qwen3-MoE: hidden 2,304, 32 query heads / 4 KV heads (GQA) at head dim 128 with QK-Norm (RMSNorm), pre-RMSNorm, SiLU-gated MLPs, RoPE base 500,000, and 64 experts per layer (expert intermediate 896) with top-8 routing, no shared expert, untied 98,304-token embeddings. Three deployment-oriented modifications distinguish it: a 3:1 sliding-window pattern (1,024-token window on three of every four layers, full attention on the fourth), a single MTP head (one extra transformer layer, loss weight 0.1) that doubles as a speculative-decoding draft, and a latency budget — matching Qwen2.5-7B on one H100 — against which every choice was ablated (dense variants, MLA, hybrids, shared experts, and auxiliary-loss-free balancing were all tried and rejected). Pre-training runs ~10.65T tokens in three phases (web 70→44→23%, code 23→42→59%) with a 50/10/50% FIM schedule, distributed Muon (Moonlight config) under BF16+FP8 hybrid precision, and a Warmup-Hold-Decay schedule with linear decay to zero. The 128K context comes from layer-selective YaRN — frequency re-mapping applied only to the global layers, per the Gemma-3/OLMo-3 recipe — over a ~117B-token extension run. Post-training splits into two variants from one long-context base: Instruct (direct answers, ~47B SFT tokens) and Thinking (explicit reasoning traces, ~167B SFT tokens, last-turn loss, turn unfolding), each refined by GRPO-style RLVR on verifiable math/code/tool tasks with IcePop truncation and a concision penalty. Released under Apache 2.0 with an unusually candid discussion of what was ablated and why.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Qwen3-MoE recipe — 'closely follows the Qwen3-MoE recipe' (MoE-in-every-layer, GQA, QK-Norm, RMSNorm, SiLU, RoPE); scaled down from Qwen3-30B-A3B", "provenance": "stated", "ref": "§2, §2.1.2 (cites Qwen3 Technical Report)" }, "origin": { "value": "Qwen3 (not in atlas)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Sliding Window Attention (3:1 pattern, 1,024-token window)", "provenance": "stated", "ref": "§2.1.4 (cites Longformer and Mistral 7B)" }, "origin": { "value": "Mistral 7B / Longformer (not in atlas)", "provenance": "stated", "ref": "§2.1.4" } }, { "mechanism": { "value": "Multi-Token Prediction head (auxiliary objective + draft model for speculative decoding)", "provenance": "stated", "ref": "§2.1.5 (cites Gloeckle et al., 2024)" }, "origin": { "value": "Gloeckle et al. (not in atlas)", "provenance": "stated", "ref": "§2.1.5" } }, { "mechanism": { "value": "Layer-selective context scaling — extension applied only to global-attention layers ('first reported in the Gemma 3 technical report… subsequently adopted by OLMo 3'); Mellum 2 uses YaRN as the scaling method", "provenance": "stated", "ref": "§4.1 (cites Gemma 3 and OLMo 3)" }, "origin": { "value": "Gemma 3 / OLMo 3", "provenance": "stated", "ref": "§4.1" } }, { "mechanism": { "value": "Muon optimizer in the distributed Moonlight configuration", "provenance": "stated", "ref": "§3.4.1 (cites Jordan et al. and 'Muon is Scalable for LLM Training')" }, "origin": { "value": "Moonlight (not in atlas)", "provenance": "stated", "ref": "§3.4.1" } }, { "mechanism": { "value": "GRPO recipe elements — token-level loss, DAPO-style clip-higher and soft overlong penalty, Dr. GRPO-style leave-one-out baseline, IcePop truncation", "provenance": "stated", "ref": "§5.2.3 (cites DAPO, Dr. GRPO, Ring-1T/IcePop)" }, "origin": { "value": "DeepSeekMath-GRPO / DAPO / Dr. GRPO (not in atlas)", "provenance": "stated", "ref": "§5.2.3" } } ], "influence_out": [], "notable_omissions": { "value": [ "Total and active parameter counts are approximate only (≈12B / ≈2.5B); no exact figure or per-component breakdown", "No training compute (FLOPs/GPU-hours) disclosed", "Tokenizer algorithm not named (custom tokenizer, shared with Mellum-4B)", "Partial-RoPE configuration not stated (RoPE base 500,000 given, dimension fraction not)", "YaRN scaling factor for the context extension not stated", "MTP head parameter count not stated (removed at inference, so deployment cost is clean, but training overhead beyond the stated 7% is not quantified)", "Pretraining batch-size ramp (2,048→4,096 sequences) is disclosed but no token-level batch in FLOPs", "The report acknowledges weaknesses: broad world knowledge (GPQA Diamond 40.9 vs 79.8 for Qwen3.5-9B), BS-Bench false-premise pushback (14-24 vs 56-70), and a HarmBench regression after RL (8.4 → 23.1)", "RULER QA subsets were depressed by a prompt-formatting issue during the extension stage; absolute long-context scores are flagged by the authors as conservative", "Only seven of eighteen MultiPL-E languages are evaluated" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "jetbrains-mellum2-thinking-12b-a2-5b.json" }, { "slug": { "value": "meituan-longcat-flash-lite-68-5b-a3b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2601.21204", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/meituan-longcat-flash-lite-68-5b-a3b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "LongCat-Flash", "provenance": "stated", "ref": "§6.1 ('LongCat-Flash-Lite adopts the same architecture as Longcat-Flash (Meituan, 2025)'); byline 'Meituan LongCat Team'" }, "variants": [], "org": { "value": "Meituan LongCat Team", "provenance": "stated", "ref": "Byline" }, "release_date": { "value": "2026-01-29", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date 2026-01-29 per export.arxiv.org API metadata (arXiv:2601.21204)" }, "report_url": { "value": "https://arxiv.org/abs/2601.21204", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "license never named (Hugging Face repo linked without license statement)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('we introduce and open-source LongCat-Flash-Lite'); HF link huggingface.co/meituan-longcat/LongCat-Flash-Lite" } }, "scale": { "total_params": { "value": 68.5, "provenance": "stated", "ref": "Abstract, §6.1" }, "active_params_per_token": { "value": 2.9, "provenance": "stated", "ref": "Abstract, §6.1", "note": "activation is dynamic: 'dynamically activates between 2.9B and 4.5B parameters per token due to the zero-experts'; abstract rounds to ~3B" }, "layers": { "value": 28, "provenance": "derived", "note": "14 shortcut layers x 2 sub-layers per shortcut (report's own equivalence: '40 shortcut layers (equivalent to 80 conventional layers)', §3.3.2 / Figure 5 caption)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "model dimension D never given a value (only symbolically in Eq. 1-3)" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "expert/dense FFN dimensions not disclosed" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive without hidden and FFN dims" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "attention configuration deferred to the Longcat-Flash technical report (Meituan 2025, arXiv 2509.01322)" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed in this report" }, "head_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in this report" }, "vocab_size": { "value": 128000, "provenance": "stated", "ref": "§3.2.2 ('n-gram vocabulary size set to 30x the base vocabulary (128k)')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "tying not disclosed" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "every shortcut layer contains an MoE module (256 FFN experts + 128 zero-experts, §6.1); no dense layers disclosed" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "§6.1 ('the MoE module consists of 256 FFN experts and 128 zero-experts')" }, "experts_per_token": { "value": 12, "provenance": "stated", "ref": "§6.1 ('each token selects 12 experts')" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "shared experts not mentioned; the 128 zero-experts are a separate always-present pool that can be dropped (dynamic activation), not described as shared experts" }, "routing": { "value": "softmax top-k with router scaling; zero-expert selection processed in a single unified kernel (Softmax + TopK + router scaling + zero-expert selection)", "provenance": "stated", "ref": "§6.4" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "load-balancing mechanism not described; zero-experts may serve a balancing/regularization role but the report does not say so" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "granularity not described" } }, "attention_variant": { "value": null, "provenance": "unknown", "note": "attention mechanism not described in this report — deferred to the Longcat-Flash technical report (Meituan 2025, arXiv 2509.01322); only attention-related kernels (splitkv combine, Q-Norm + KV-Norm) appear in §6.4" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "not disclosed in this report" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: residual-branch ('shortcut') architecture with pre-norm identity connections (§3.2.4, Figure 5); no learned attention over prior layer outputs" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "inferred", "note": "YaRN (§6.1) is defined as an extension of RoPE models, implying a RoPE backbone; the report never names RoPE explicitly" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": false, "provenance": "inferred", "note": "Absence rule: no partial-RoPE mention" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "ref": "§6.1 ('we implement YARN (Peng et al., 2023) during the 32k sequence length training stage, enabling LongCat-Flash-Lite to handle sequences up to 256k tokens')" }, "factor": { "value": null, "provenance": "unknown", "note": "YaRN factor not disclosed" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§6.1 ('handle sequences up to 256k tokens')" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§6.4 (kernel fusion: 'AllReduce + Residual Add + RMSNorm')" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§3.3.2 ('For pre-normalization architectures, the contribution of N-gram Embedding through the identity connection (residual branch) inherently diminishes as network depth increases')" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§6.4 (kernel fusion: 'AllGather + Q-Norm + KV-Norm')", "note": "report names Q-Norm and KV-Norm in the model's fused attention kernels; the attention mechanism itself is deferred to the Longcat-Flash report, so this is taken at face value as QK-normalisation on the attention path" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§5.1 Eq. 4 / §5.2 Eq. 5 (FFN output Wd(SiLU(Wg x) ⊙ e) — SiLU-gated FFN in the PLE/PLNE formulation of the model's MLP)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink mechanism described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no logit softcapping described" }, "other": [ { "value": "Embedding Amplification: embedding output scaled by sqrt(D) or LayerNormed before merging with the residual branch (from Takase et al. 2025) — prevents the first attention module's output from drowning out the embedding signal in the residual stream", "provenance": "stated", "ref": "§3.2.4" }, { "value": "Zero-experts: 128 zero-expert slots per MoE layer whose selection lets per-token activation vary between 2.9B and 4.5B parameters", "provenance": "stated", "ref": "§6.1" } ] } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§6.1 ('mid-training during which the sequence length is extended to 128k'; pre-training itself ran at 8k)" }, "deployed_length": { "value": 262144, "provenance": "stated", "ref": "§6.1 ('handle sequences up to 256k tokens')" }, "extension_method": { "value": "YaRN implemented during the 32k sequence-length training stage, enabling up to 256k; training curriculum: 8k pre-training (11T) -> 128k mid-training (1.5T) -> SFT", "provenance": "stated", "ref": "§6.1" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer algorithm not described" }, "notes": { "value": null, "provenance": "unknown", "note": "only the 128k base vocabulary size is given (§3.2.2)" } }, "training": { "tokens": { "value": 12.5, "provenance": "derived", "note": "11T pre-training + 1.5T mid-training = 12.5T (both stated in §6.1; SFT data volume not counted)" }, "data_composition": { "value": "same data recipe as LongCat-Flash-Chat (Meituan, 2025); not detailed in this report", "provenance": "stated", "ref": "§6.1" }, "curriculum": { "value": "three stages: pre-training on 11T tokens at 8k sequence length, mid-training on 1.5T tokens with sequence length extended to 128k, then supervised fine-tuning", "provenance": "stated", "ref": "§6.1" }, "optimizer": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "batch_schedule": { "value": "batch size increased at 420B tokens (Figure 10 caption: 'The loss drop at 420B tokens coincides with the batch size increases')", "provenance": "stated", "ref": "Figure 10 caption" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not stated; a quantized deployment exists ('For the quantized model, we integrate every activation quantization step into existing operators', §6.4) but the format is unnamed" }, "parallelism": { "value": "wide EP (Expert Parallel) and SBO (Single Batch Overlap) adopted for inference deployment", "provenance": "stated", "ref": "§6.4", "note": "training parallelism not disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed; decoding benchmark used 8xH800-80G (Figure 8 caption)" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§6.1 ('finally trained on SFT data'), §6 ('complete pipeline of pre-training, mid-training, and supervised finetuning')" }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "report describes no RL or preference-optimization stage" }, "reasoning_training": { "value": "none disclosed", "provenance": "stated", "ref": "no reasoning-specific training described beyond SFT" }, "distillation": { "value": "none (trained from scratch)", "provenance": "inferred", "note": "report states the model is 'trained from scratch' (Abstract, §6) with no teacher distillation; N-gram Embedding is a parameter-allocation mechanism, not distillation" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "no non-text modality described; all evaluations are text tasks" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": null, "provenance": "unknown", "note": "attention variant not disclosed in this report, so KV-cache consequences cannot be derived; splitkv-and-combine decoding kernels exist (§6.4)" }, "quantization": { "value": "quantized deployment with activation quantization fused into operators; quantization format not named", "provenance": "stated", "ref": "§6.4" }, "speculative_dedup": { "value": "Eagle3 with a 3-step speculative decoding strategy; draft model uses a conventional embedding layer (no n-gram lookup) and n-gram embeddings are cached during drafting to avoid redundant verification-time computation", "provenance": "stated", "ref": "§6.4, §4.2" }, "serving": { "value": "wide EP + SBO; extensive kernel fusion (AllReduce+Residual Add+RMSNorm; AllGather+Q-Norm+KV-Norm; ReduceScatter+RMSNorm+Hidden State Combine; router Softmax+TopK+scaling+zero-expert selection); optimized splitkv-and-combine attention kernel (-50% combine latency); PDL (Programmatic Dependent Launch); N-gram Cache with custom CUDA kernels", "provenance": "stated", "ref": "§6.4, §4.2" } }, "evaluation": { "benchmarks": [ { "name": { "value": "Tau2-Airline (avg@8)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 58.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Tau2-Retail (avg@8)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Tau2-Telecom (avg@8)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 72.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "VitaBench (avg@4)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 7.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SWE-Bench (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 54.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "TerminalBench (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 33.75, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SWE-Bench Multilingual", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 38.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "PRDBench", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 39.63, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA-Diamond (avg@16)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 66.78, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 85.52, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU-Pro (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 78.29, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "CEval (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 86.55, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "CMMLU (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 82.48, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MATH500 (acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 96.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AIME24 (avg@32)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 72.19, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AIME25 (avg@32)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 63.23, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BBH (base, @1.3T)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 43.67, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "DROP (base, @1.3T)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 52.43, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "GSM8K (base, @1.3T)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 50.5, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "BigCodeBench (base, @1.3T)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 36.05, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Embedding scaling as an orthogonal sparsity dimension: N-gram Embedding achieves a superior Pareto frontier vs expert scaling in high-sparsity regimes (crossover near total/active ratio ~20; wider models sustain the advantage to ratios beyond 50)", "provenance": "stated", "ref": "§1, §3.1, §3.3" }, "ref": { "value": "§1, §3.1, §3.3", "provenance": "stated", "ref": "§1, §3.1, §3.3" } }, { "text": { "value": "Systematic characterization of the architectural factors governing embedding-scaling efficacy: integration timing, parameter budgeting, hash-collision mitigation via vocabulary sizing, n-gram order/sub-table hyperparameters, Embedding Amplification, and width/depth interplay", "provenance": "stated", "ref": "§3" }, "ref": { "value": "§3", "provenance": "stated", "ref": "§3" } }, { "text": { "value": "LongCat-Flash-Lite: 68.5B model with 31.4B N-gram Embedding parameters (46% of total) and dynamic 2.9-4.5B activation via 128 zero-experts; outperforms the parameter-equivalent MoE baseline and is competitive with Qwen3-Next-80B-A3B, Kimi-Linear-48B-A3B and Gemini 2.5 Flash-Lite, especially in agentic tool use and coding", "provenance": "stated", "ref": "§6" }, "ref": { "value": "§6", "provenance": "stated", "ref": "§6" } }, { "text": { "value": "Inference efficiency: N-gram Cache and synchronized kernels eliminate embedding-lookup overhead; N-gram Embedding reduces MoE memory-I/O in decoding; synergy with speculative decoding (Eagle3, 3-step) via draft-side conventional embeddings and embedding caching", "provenance": "stated", "ref": "§4, §6.4" }, "ref": { "value": "§4, §6.4", "provenance": "stated", "ref": "§4, §6.4" } } ], "assessment": { "value": "The genuinely novel axis here is parameter allocation: instead of more experts or a new mixing layer, LongCat-Flash-Lite pushes 46% of its budget into a hashed n-gram embedding table (over 30B parameters), a mechanism with O(1) lookup that never touches compute. The N-gram Embedding mechanism itself is borrowed (Clark et al. 2022 / Huang et al. 2025 Over-Encoding), and the report's contribution is the scaling analysis plus system engineering. vs the atlas: the closest relative is moonshot-kimi-linear-48b-a3b (48B-A3B MoE, long context, agentic-leaning), but that model scales via linear attention while LongCat-Flash-Lite scales via embeddings and dynamic zero-expert activation — a fundamentally different sparsity strategy; the 256K YaRN extension and 128k mid-training confirm the LongCat long-context family angle. The 'superior Pareto frontier' claim rests on in-house 300B-token experiments with intersection points that shift with width — honest but not independently verifiable. Weaknesses: the attention mechanism, hidden dim, heads, optimizer, and training precision are all deferred or undisclosed, so the architecture is only partially documented in this report.", "provenance": "inferred", "note": "comparison against atlas entries (moonshot-kimi-linear-48b-a3b, deepseek-v3, zai-glm-4-5-air) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all shortcut layers carry an MoE module (256 experts + 128 zero-experts, top-12, §6.1)" }, "sequence_mixing": { "value": null, "provenance": "unknown", "note": "attention mechanism not disclosed in this report (deferred to Longcat-Flash, arXiv 2509.01322), so the layer pattern cannot be classified" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no multimodal content in report)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (open-sourced with HF repo); data recipe referenced but not released" }, "scale_class": { "value": "large", "provenance": "derived", "note": "68.5B (10B <= 68.5B < 100B)" }, "design_intent": { "value": "long-context", "provenance": "inferred", "note": "LongCat family name and 256K context via YaRN (128K mid-training) anchor the long-context intent; report also emphasizes agentic/coding strength and inference efficiency (secondary intents in prose). Research framing of the report itself is embedding scaling." } }, "prose": { "design_overview": { "value": "LongCat-Flash-Lite is Meituan's demonstration that parameter budget can be moved out of the MoE layers entirely. The 68.5B-parameter model spends 31.4B of them — 46% of the total — on an N-gram Embedding module: each token's embedding is the average of a base 128K-vocab table plus hashed n-gram sub-tables (polynomial rolling hash, K sub-tables with linear projections back into the model space), so capacity grows with vocabulary rather than compute. Because the extra parameters are looked up in O(1) rather than routed, the model keeps total size while cutting activated parameters, which the authors show pays off most at high sparsity: their scaling study (280M/790M/1.3B activated probes over 300B tokens) finds N-gram Embedding beats parameter-equivalent expert scaling once the total/active ratio exceeds ~20, with the crossover shifting higher in wider models and shrinking in deeper ones. On top of the Longcat-Flash MoE backbone (14 shortcut layers, each with two sub-layers; 256 FFN experts plus 128 zero-experts, top-12), the report adds Embedding Amplification — scaling the embedding output by sqrt(D) before the residual merge so the first attention module does not drown it out. Context reaches 256K via YaRN at the 32K stage and a 128K mid-training phase. Inference leans on the reduced activation: Eagle3 3-step speculative decoding, wide expert parallelism, heavy kernel fusion (including Q-Norm + KV-Norm and a unified router kernel), a specialized N-gram Cache, and PDL overlap. The report defers the attention mechanism itself to the Longcat-Flash technical report, so the sequence-mixing design is not documented here.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "N-gram Embedding (Over-Encoding): vocabulary-free n-gram tables with K sub-tables and linear projections", "provenance": "stated", "ref": "§2 (cites Clark et al. 2022; Huang et al. 2025; Pagnoni et al. 2025)" }, "origin": { "value": "Over-Tokenized Transformer / CANINE (not in atlas)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Embedding Amplification (embedding output scaling / LayerNorm before residual merge)", "provenance": "stated", "ref": "§3.2.4 (cites Takase et al. 2025)" }, "origin": { "value": "Takase et al. 2025 (not in atlas)", "provenance": "stated", "ref": "§3.2.4" } }, { "mechanism": { "value": "YaRN context extension", "provenance": "stated", "ref": "§6.1 (cites Peng et al. 2023)" }, "origin": { "value": "YaRN (not in atlas; used by Qwen et al.)", "provenance": "stated", "ref": "§6.1" } }, { "mechanism": { "value": "Eagle3 speculative decoding", "provenance": "stated", "ref": "§6.4 (cites Li et al. 2025)" }, "origin": { "value": "EAGLE-3 (not in atlas)", "provenance": "stated", "ref": "§6.4" } } ], "influence_out": [], "notable_omissions": { "value": [ "Attention mechanism entirely deferred to the Longcat-Flash technical report (arXiv 2509.01322) — attention variant, heads, layer pattern, and KV-cache design are undisclosed here", "Hidden dimension and all FFN/expert dimensions not disclosed", "Optimizer, learning-rate schedule, and training precision not disclosed", "Training hardware and compute (FLOPs/GPU-hours) not disclosed", "Load-balancing mechanism for the 256+128 expert pools not described", "N-gram hyperparameters for the final model (N, K, n-gram vocabulary size) not given (only 30x-33x base-vocab guidance from ablations)", "No license named", "Embedding tying not disclosed", "Evaluation table values for Gemini 2.5 Flash-Lite and some rows are sourced from public reports (marked with *), not measured in-house", "No RL/post-SFT pipeline at all (SFT only), unlike peers of this scale" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "meituan-longcat-flash-lite-68-5b-a3b.json" }, { "slug": { "value": "meta-llama-3.1", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2407.21783", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/meta-llama-3.1", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Llama 3.1", "provenance": "stated", "ref": "§1 (report title is 'The Llama 3 Herd of Models'; the released family is named Llama 3.1)" }, "variants": [ { "value": "8B", "provenance": "stated", "ref": "Table 1" }, { "value": "70B", "provenance": "stated", "ref": "Table 1" }, { "value": "405B", "provenance": "stated", "ref": "Table 1" } ], "org": { "value": "Meta (Llama Team, AI @ Meta)", "provenance": "stated", "ref": "Title block" }, "release_date": { "value": "2024-07-31", "provenance": "inferred", "note": "arXiv v1 submission date (ID from report URL; arXiv API metadata) — report text itself is undated" }, "report_url": { "value": "https://arxiv.org/abs/2407.21783", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Llama 3 Community License", "provenance": "stated", "ref": "§1", "note": "Report: released 'under an updated version of the Llama 3 Community License; see https://llama.meta.com'." }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract / §1", "note": "Report: 'We publicly release Llama 3, including pre-trained and post-trained versions of the 405B parameter language model and our Llama Guard 3 model'." } }, "scale": { "total_params": { "value": 405, "provenance": "stated", "ref": "Abstract / §1 / Table 3" }, "active_params_per_token": { "value": 405, "provenance": "derived", "note": "Dense model: active params per token = total params (405B); report states a dense Transformer (§1, §3.2)." }, "layers": { "value": 126, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 16384, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 53248, "provenance": "stated", "ref": "Table 3" }, "ffn_ratio": { "value": 3.3, "provenance": "derived", "note": "53248 / 16384 = 3.25, rounded to one decimal." }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3 / §3.2" }, "head_dim": { "value": 128, "provenance": "derived", "note": "hidden_dim / attention_heads_q = 16384 / 128 = 128." }, "vocab_size": { "value": 128000, "provenance": "stated", "ref": "Table 3 / §3.2" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Report silent on input/output embedding tying; not part of the disclosed configuration." }, "variants": [ { "name": { "value": "8B", "provenance": "stated", "ref": "Table 1 / Table 3" }, "total_params": { "value": 8, "provenance": "stated", "ref": "§1" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 14336, "provenance": "stated", "ref": "Table 3" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3" }, "context_length": { "value": 131072, "provenance": "inferred", "note": "Table 1 marks Llama 3.1 8B as long-context; 128K stated for the family (§3.4.2); §3.4 says similar pre-training recipes were used for 8B and 70B." } }, { "name": { "value": "70B", "provenance": "stated", "ref": "Table 1 / Table 3" }, "total_params": { "value": 70, "provenance": "stated", "ref": "§1" }, "layers": { "value": 80, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 8192, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 28672, "provenance": "stated", "ref": "Table 3" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3" }, "context_length": { "value": 131072, "provenance": "inferred", "note": "Table 1 marks Llama 3.1 70B as long-context; 128K stated for the family (§3.4.2); §3.4 says similar pre-training recipes were used for 8B and 70B." } }, { "name": { "value": "405B", "provenance": "stated", "ref": "Table 1 / Table 3" }, "total_params": { "value": 405, "provenance": "stated", "ref": "§1" }, "layers": { "value": 126, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 16384, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 53248, "provenance": "stated", "ref": "Table 3" }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§3.4.2 / Table 4" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "stated", "ref": "Abstract / §1 / §3.2", "note": "Report: 'standard dense Transformer' chosen 'rather than for a mixture-of-experts model'." }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§1 / §3.2" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§1 / §3.2" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§1 / §3.2" }, "routing": { "value": null, "provenance": "n/a", "ref": "§1 / §3.2" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§1 / §3.2" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§1 / §3.2" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§3.2", "note": "Report: 'We use grouped query attention (GQA) with 8 key-value heads'." }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "Report describes a standard dense Transformer with GQA and no per-layer attention variation (§3.2); document-level attention mask affects training data, not the layer pattern." }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "Table 3 / §3.2" }, "rope_base": { "value": 500000, "provenance": "stated", "ref": "Table 3 / §3.2" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "Report discusses only the RoPE base frequency; silent on how many dimensions RoPE is applied to." }, "extension": { "method": { "value": "other", "provenance": "stated", "ref": "§3.4.2", "note": "Context extended by continued pre-training on progressively longer sequences (six stages, 8K to 128K); no interpolation/scaling method (YaRN/NTK/PI) disclosed." }, "factor": { "value": null, "provenance": "unknown", "note": "Report discloses no extension scaling factor." }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§3.4.2 / Table 4" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "Report never names its normalization (no RMSNorm/LayerNorm mention anywhere)." }, "placement": { "value": null, "provenance": "unknown", "note": "Report silent on pre/post placement of normalization." }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: §3.2 enumerates all architectural changes vs Llama 2; QK-normalisation is absent from that list and never mentioned." } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "Table 3" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Report designs no sink tokens; the architecture description would surface a designed sink mechanism were it used. Any attention-sink behaviour is emergent." }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: logit softcapping never mentioned; absent from the §3.2 enumeration of architectural changes." }, "other": { "value": [], "provenance": "inferred", "note": "Report describes no additional block-level stability mechanisms; its stability discussion covers training numerics (FP32 gradient accumulation, §3.3.2) and FP8 inference mitigations (§6.2), not the block." } }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: report describes its block in detail and never mentions attention over prior layer outputs; standard sequential residual connections are the default mechanism." } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§3.4.2 / Table 4", "note": "Report: long-context stage ends at a 128K token context window; Table 4 lists sequence length 131,072." }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "Abstract / §2", "note": "Report: 'context window of up to 128K tokens'." }, "extension_method": { "value": "other: continued pre-training (staged 8K to 128K); no interpolation method disclosed", "provenance": "stated", "ref": "§3.4.2" } }, "tokenizer": { "algorithm": { "value": "tiktoken-based (100K tiktoken tokens + 28K additional tokens)", "provenance": "stated", "ref": "§3.2" }, "notes": { "value": "Vocabulary combines 100K tokens from the tiktoken tokenizer with 28K additional tokens for better non-English support; improves English compression from 3.17 to 3.94 characters per token vs the Llama 2 tokenizer. Vocab size (128,000) lives in scale.vocab_size.", "provenance": "stated", "ref": "§3.2" } }, "training": { "tokens": { "value": 15.6, "provenance": "stated", "ref": "§1", "note": "Report: flagship 'pre-trained ... on 15.6T text tokens'; corpus described as 'about 15T multilingual tokens'." }, "data_composition": { "value": "Final mix roughly 50% general knowledge, 25% mathematical/reasoning, 17% code, 8% multilingual tokens; heavy curation of web data (PII/safety filtering, URL/doc MinHash and line-level dedup, heuristic and model-based quality filtering, code/math pipelines, 176-language LID); knowledge cutoff end of 2023.", "provenance": "stated", "ref": "§3.1.1 / §3.1.2" }, "curriculum": { "value": "Three stages: (1) initial pre-training at 8K context, (2) long-context pre-training to 128K in six increments (~800B tokens), (3) annealing on the final 40M tokens with LR linearly annealed to 0 and checkpoint averaging.", "provenance": "stated", "ref": "§3.4" }, "optimizer": { "value": "AdamW", "provenance": "stated", "ref": "§3.4.1" }, "lr_schedule": { "value": "Peak LR 8e-5 (405B; 3e-4 for 8B and 1.5e-4 for 70B per Table 3), linear warmup of 8,000 steps, cosine decay to 8e-7 over 1,200,000 steps.", "provenance": "stated", "ref": "§3.4.1 / Table 3" }, "batch_schedule": { "value": "Starts at 4M tokens (seq len 4,096); doubled to 8M tokens at seq len 8,192 after 252M tokens; doubled again to 16M after 2.87T tokens.", "provenance": "stated", "ref": "§3.4.1" }, "precision": { "value": "BF16 (38-43% BF16 MFU); FP32 gradient accumulation and FP32 reduce-scatter for numerical stability", "provenance": "stated", "ref": "§3.3.2" }, "parallelism": { "value": "4D parallelism combining tensor, pipeline, context, and data parallelism (FSDP), ordered [TP, CP, PP, DP]; all-gather-based CP; interleaved PP schedule with tunable micro-batch count N; network-aware configuration.", "provenance": "stated", "ref": "§3.3.2" }, "hardware": { "value": "Up to 16K H100 GPUs (700W TDP, 80GB HBM3) on Meta Grand Teton servers (8 GPUs/server, NVLink); RoCE fabric (Arista 7800, Minipack2) for 405B, Nvidia Quantum2 InfiniBand for smaller models; 400 Gbps interconnects; MAST scheduler; Tectonic storage fabric.", "provenance": "stated", "ref": "§3.3.1" }, "compute": { "value": "3.8e25 FLOPs (405B pre-training), almost 50x the largest Llama 2 model", "provenance": "stated", "ref": "§1" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4 / §4.1.3", "note": "Six iterative rounds; each round is SFT followed by DPO. SFT data: rejection-sampled human prompts, synthetic capability data, small amounts of human-curated data; 405B finetuned at LR 1e-5 over 8.5K-9K steps." }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "§4.1.4", "note": "DPO with beta 0.1, formatting-token masking and an added NLL term (coefficient 0.2); report says PPO was explored but DPO needed less compute and performed better at scale." }, "reasoning_training": { "value": "SFT on step-wise reasoning traces filtered by answer correctness, self-verification, and outcome/stepwise reward models (MCTS used for challenging prompts); interleaved text+Python-code reasoning with execution feedback; no RLVR or test-time compute disclosed.", "provenance": "stated", "ref": "§4.3.3" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "Report describes no distillation from a teacher model; 405B-generated SFT data for 8B/70B is teacher-generated training data, not distillation (schema §8). Code/multilingual 'experts' are continued pre-training branches (§4.3.1, §4.3.2), not distilled models." } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes image/video/speech extensions via compositional cross-attention adapters but states they are 'still under development and not yet ready for release' (§2, §7); the released herd is a language-model family (Abstract)." }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "Abstract / §2", "note": "Multimodal attachments (image encoder + cross-attention adapters) exist only in unreleased experimental models." } }, "efficiency": { "kv_cache": { "value": "GQA with 8 KV heads: KV-cache size and decoding memory reduced vs MHA; all-gather CP overhead small because KV tensors are far smaller than Q under GQA", "provenance": "stated", "ref": "§3.2 / §3.3.2" }, "quantization": { "value": "FP8 inference for 405B (FFN-layer matmuls only, row-wise dynamic scaling, scaling factors upper-bounded at 1200, first/last layers unquantized; kernels released); Llama Guard 3 ships an int8-quantized variant (size reduced >40%)", "provenance": "stated", "ref": "§6.2 / §5.4.7" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "Report does not mention speculative decoding, draft models, or multi-token prediction." }, "serving": { "value": "405B inference: BF16 pipeline parallelism across 16 GPUs on two machines (TP within node over NVLink, PP across nodes) with micro-batching for better throughput-latency trade-off; FP8 inference raises pre-fill throughput up to 50% vs two-machine BF16", "provenance": "stated", "ref": "§6.1 / §6.2" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (5-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 87.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU-Pro (5-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "IFEval", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HumanEval (0-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 89.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MBPP EvalPlus (0-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GSM8K (8-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 96.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MATH (0-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ARC Challenge (0-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 96.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA (0-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 51.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BFCL", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Nexus", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 58.7, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ZeroSCROLLS/QuALITY", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 95.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "InfiniteBench/En.MC", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 83.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "NIH/Multi-needle", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 98.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MGSM (0-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 91.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Data as a primary lever: more careful pre-processing and curation pipelines for pre-training data and rigorous quality assurance/filtering for post-training data, yielding a ~15T-token corpus vs 1.8T for Llama 2.", "provenance": "stated", "ref": "§1 / §3.1" }, "ref": { "value": "§1 / §3.1", "provenance": "stated", "ref": "§1 / §3.1" } }, { "text": { "value": "Scale: a 405B dense flagship pre-trained on 15.6T tokens with 3.8e25 FLOPs (~50x the largest Llama 2), with smaller models over-trained past compute-optimality to be stronger at the same inference budget.", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } }, { "text": { "value": "Managing complexity: a deliberately standard dense Transformer (rather than MoE) and a simple, scalable post-training recipe of supervised finetuning, rejection sampling, and DPO rather than heavier RL algorithms.", "provenance": "stated", "ref": "§1 / §4" }, "ref": { "value": "§1 / §4", "provenance": "stated", "ref": "§1 / §4" } }, { "text": { "value": "128K-token context support achieved by staged continued pre-training with an increased RoPE base (500,000) and a document-separation attention mask important for long-sequence training.", "provenance": "stated", "ref": "§3.2 / §3.4.2" }, "ref": { "value": "§3.2 / §3.4.2", "provenance": "stated", "ref": "§3.2 / §3.4.2" } }, { "text": { "value": "A compositional approach to multimodal extension (image, video, speech) via cross-attention adapters over the frozen LLM, competitive with state of the art without joint multimodal pre-training (experimental, not released).", "provenance": "stated", "ref": "Abstract / §7" }, "ref": { "value": "Abstract / §7", "provenance": "stated", "ref": "Abstract / §7" } }, { "text": { "value": "Open release of pre-trained and post-trained 405B models plus the Llama Guard 3 system-level safety suite (Prompt Guard, Code Shield), intended to spur research.", "provenance": "stated", "ref": "§1 / §5.4.7" }, "ref": { "value": "§1 / §5.4.7", "provenance": "stated", "ref": "§1 / §5.4.7" } } ], "assessment": { "value": "Architecturally Llama 3.1 is deliberately incremental: the report itself frames the block as a standard dense Transformer barely diverging from Llama 2, with GQA, a raised RoPE base, a tiktoken-derived vocabulary, and a document mask as the only enumerated changes — no new attention, normalization, or stability mechanisms relative to peers. Its genuine novelty lies outside the block: an order-of-magnitude jump in data and compute (15.6T tokens, 3.8e25 FLOPs), context extension by staged continued pre-training rather than post-hoc interpolation, and an iterated SFT + rejection-sampling + DPO pipeline that scaled to a 405B open release with a system-level safety suite. The multimodal adapters follow Flamingo-style cross-attention (cited in-report) and are explicitly unreleased, so they do not count as a shipped contribution.", "provenance": "inferred", "note": "authored by atlas analyst (comparison against other atlas entries)" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "Report states a standard dense Transformer with no MoE (§1, §3.2); all FFN layers dense." }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "Uniform full attention with GQA in every layer; no restricted/sparse/state-space patterns described (§3.2)." }, "modality": { "value": "text-only", "provenance": "derived", "note": "From modality.type: released family is text-only; multimodal extensions unreleased (Abstract, §2)." }, "openness": { "value": "open-weights", "provenance": "derived", "note": "Report states public release of weights (§1) and does not disclose release of the training corpus (only coarse mix percentages, §3.1.2)." }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "Reference variant 405B >= 100B." }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "Report frames the herd as general-purpose foundation models natively supporting multilinguality, coding, reasoning, and tool use, evaluated across a broad benchmark sweep (Abstract, §1, Table 2)." } }, "prose": { "design_overview": { "value": "Llama 3.1 is a deliberately conservative dense Transformer: the report describes a standard architecture (Vaswani et al. 2017) that 'does not deviate significantly' from Llama 2, with four enumerated changes — grouped-query attention with 8 KV heads (shrinking KV caches during decoding), a document-level attention mask that blocks cross-document mixing within a sequence, a 128K-token vocabulary (100K tiktoken tokens plus 28K added for non-English), and a RoPE base frequency raised to 500,000 for longer contexts. The 405B reference variant stacks 126 layers of hidden size 16,384 with 128 query heads, an FFN of 53,248, and SwiGLU activations; head dimension is 128. Context grows from 8K to 128K tokens not by interpolation but by staged continued pre-training, followed by an annealing phase with checkpoint averaging. The pre-training recipe is plain but extreme in scale: AdamW, cosine schedule, batch size doubling from 4M to 16M tokens, 15.6T tokens and 3.8e25 FLOPs, run with 4D parallelism (TP/CP/PP/DP) on up to 16K H100s with a tuned interleaved pipeline schedule. Post-training runs six iterative rounds of SFT and DPO, leaning heavily on rejection sampling, synthetic data, and reward-model filtering, with dedicated capability pushes for code, multilinguality, math, long context, tool use, and factuality. The report also sketches compositional image/video/speech extensions via cross-attention adapters, but these are explicitly unreleased; the shipped herd is text-only.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "standard dense Transformer (self-attention + FFN blocks)", "provenance": "inferred", "note": "authored by atlas analyst; report cites Vaswani et al. 2017 (§3.2)" }, "origin": { "value": "Transformer (Vaswani et al. 2017)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "overall architecture continuity with the Llama line", "provenance": "inferred", "note": "authored by atlas analyst; report: 'does not deviate significantly from Llama and Llama 2' (§3.2)" }, "origin": { "value": "Llama / Llama 2", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "grouped-query attention with 8 KV heads", "provenance": "inferred", "note": "authored by atlas analyst; report cites ainslie2023gqa (§3.2)" }, "origin": { "value": "GQA (Ainslie et al. 2023)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "RoPE rotary position embeddings (base raised to 500,000)", "provenance": "inferred", "note": "authored by atlas analyst; report frames the base as increased relative to prior Llama models (§3.2)" }, "origin": { "value": "Llama lineage (RoPE)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "tiktoken tokenizer (100K tokens) extended with 28K tokens", "provenance": "inferred", "note": "authored by atlas analyst; report names tiktoken with URL (§3.2)" }, "origin": { "value": "OpenAI tiktoken", "provenance": "inferred", "note": "authored by atlas analyst" } } ], "influence_out": [], "notable_omissions": { "value": [ "Normalization type and placement never named (no RMSNorm/pre-norm disclosure).", "Embedding tying not disclosed.", "RoPE dimensional scope (full vs partial) not stated.", "QK-normalisation and logit softcapping never mentioned.", "Training corpus not released; only coarse mix percentages (roughly 50/25/17/8) disclosed.", "No speculative decoding or multi-token prediction disclosed.", "Per-stage token budgets of the six long-context stages not itemised (only ~800B total for the stage).", "Multimodal models described in detail but explicitly not released." ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "meta-llama-3.1.json" }, { "slug": { "value": "meta-llama-3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2407.21783", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/meta-llama-3", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Llama 3", "provenance": "stated", "ref": "Title / §1" }, "variants": [ { "value": "8B", "provenance": "stated", "ref": "Table 1" }, { "value": "70B", "provenance": "stated", "ref": "Table 1" }, { "value": "405B", "provenance": "stated", "ref": "Table 1" } ], "org": { "value": "Llama Team, AI @ Meta", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2024-07-23", "provenance": "stated", "ref": "Title block date line; note: arXiv header shows v3 dated 2024-11-23; the atlas's meta-llama-3.1 record uses arXiv v1 2024-07-31 as inferred" }, "report_url": { "value": "https://arxiv.org/abs/2407.21783", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Llama 3 Community License (updated version)", "provenance": "stated", "ref": "§1.2 / p.2 release paragraph" }, "open_weights": { "value": true, "provenance": "stated", "ref": "§1.2: 'We are publicly releasing all three Llama 3 models…'; release paragraph" } }, "scale": { "total_params": { "value": 405, "provenance": "stated", "ref": "Abstract / §1 / Table 3" }, "active_params_per_token": { "value": 405, "provenance": "derived", "note": "Dense model: active = total (405B); report states a dense Transformer (§1, §3.2)" }, "layers": { "value": 126, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 16384, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 53248, "provenance": "stated", "ref": "Table 3" }, "ffn_ratio": { "value": 3.25, "provenance": "derived", "note": "53248 / 16384 = 3.25" }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3 / §3.2 (GQA with 8 KV heads)" }, "head_dim": { "value": 128, "provenance": "derived", "note": "16384 / 128 = 128" }, "vocab_size": { "value": 128000, "provenance": "stated", "ref": "§3.2 / Table 3 (report prints 128,000; tokenizer combines 100K tiktoken tokens + 28K additional)" }, "embedding_tied": { "provenance": "unknown" }, "variants": [ { "name": { "value": "8B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 8, "provenance": "stated", "ref": "§1" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 14336, "provenance": "stated", "ref": "Table 3" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3" }, "context_length": { "value": 131072, "provenance": "inferred", "note": "Table 4 long-context stage; staged 8K→128K continued pretraining for all models (§3.4.2)" } }, { "name": { "value": "70B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 70, "provenance": "stated", "ref": "§1" }, "layers": { "value": 80, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 8192, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 28672, "provenance": "stated", "ref": "Table 3" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3" }, "context_length": { "value": 131072, "provenance": "inferred", "note": "Table 4 long-context stage; staged 8K→128K continued pretraining for all models (§3.4.2)" } }, { "name": { "value": "405B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 405, "provenance": "stated", "ref": "§1" }, "layers": { "value": 126, "provenance": "stated", "ref": "Table 3" }, "hidden_dim": { "value": 16384, "provenance": "stated", "ref": "Table 3" }, "ffn_inner_dim": { "value": 53248, "provenance": "stated", "ref": "Table 3" }, "attention_heads_q": { "value": 128, "provenance": "stated", "ref": "Table 3" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 3" }, "context_length": { "value": 131072, "provenance": "inferred", "note": "Table 4 long-context stage; staged 8K→128K continued pretraining for all models (§3.4.2)" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "inferred", "note": "Absence rule: report describes the dense Transformer block in detail (§3.2) with no routing" }, "moe": { "expert_count": { "provenance": "n/a", "ref": "§3.2 dense block" }, "experts_per_token": { "provenance": "n/a", "ref": "§3.2 dense block" }, "shared_experts": { "provenance": "n/a", "ref": "§3.2 dense block" }, "routing": { "provenance": "n/a", "ref": "§3.2 dense block" }, "load_balancing": { "provenance": "n/a", "ref": "§3.2 dense block" }, "expert_granularity": { "provenance": "n/a", "ref": "§3.2 dense block" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§3.2" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "Absence rule: no per-layer attention pattern disclosed; document-separation mask applied uniformly" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard residual connections (§3.2)" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§3.2 / Table 3" }, "rope_base": { "value": 500000, "provenance": "stated", "ref": "§3.2 / Table 3" }, "partial_rope": { "provenance": "unknown" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "Staged continued pretraining 8K→128K, no interpolation (§3.4.2)", "ref": "§3.4.2" }, "factor": { "provenance": "unknown" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "Table 4" } } }, "normalization": { "type": { "provenance": "unknown" }, "placement": { "provenance": "unknown" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: block described in detail (§3.2), no QK-normalisation mentioned" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§3.2 / Table 3" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule" }, "other": { "value": [ "document-separation attention mask (§3.2)" ], "provenance": "stated", "ref": "§3.2" } } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§3.4.2 / Table 4 (staged continued pretraining to 128K)" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§3.4.2 (128K context reported for the models)" }, "extension_method": { "value": "staged continued pretraining 8K→128K, no interpolation", "provenance": "stated", "ref": "§3.4.2" } }, "tokenizer": { "algorithm": { "value": "tiktoken-based BPE", "provenance": "stated", "ref": "§3.2" }, "notes": { "value": "Vocabulary combines 100K tokens from the tiktoken tokenizer with 28K additional tokens for non-English support; improved compression vs Llama 2 tokenizer (§3.2)", "provenance": "stated", "ref": "§3.2" } }, "training": { "tokens": { "value": 15.6, "provenance": "stated", "ref": "§1 / §3.4" }, "data_composition": { "value": "~50% general knowledge, ~25% math/reasoning, ~17% code, ~8% multilingual (§3.1.2); 4.7T tokens of synthetic data for post-training (§4)", "provenance": "stated", "ref": "§3.1.2 / §4" }, "curriculum": { "value": "Three-stage: general pretraining; annealing (40M tokens); continued pretraining 8K→128K (~800B tokens) (§3.4)", "provenance": "stated", "ref": "§3.4" }, "optimizer": { "value": "AdamW", "provenance": "stated", "ref": "Table 3" }, "lr_schedule": { "value": "Cosine schedule with peak LR 8e-5 (8B), 1.5e-4 (70B), 3e-4 (405B), decaying to 10% of peak", "provenance": "stated", "ref": "Table 3" }, "batch_schedule": { "value": "Fixed batch per compute scale, 4M→8M→16M tokens across stages", "provenance": "stated", "ref": "§3.4 / Table 4" }, "precision": { "value": "BF16 with FP32 gradient accumulation", "provenance": "stated", "ref": "§3.3" }, "parallelism": { "value": "4D parallelism: TP/CP/PP/DP with FSDP", "provenance": "stated", "ref": "§3.3" }, "hardware": { "value": "Up to 16,384 H100 GPUs (Grand Teton cluster)", "provenance": "stated", "ref": "§3.3" }, "compute": { "value": "3.8e25 FLOPs (405B pretraining)", "provenance": "stated", "ref": "Table 3" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4 (SFT data; six rounds of post-training)" }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "§4 (DPO with β=0.1, formatting-token masking, NLL term 0.2)" }, "reasoning_training": { "value": "Rejection-sampled SFT data for reasoning; no RLVR disclosed", "provenance": "inferred", "note": "§4 describes rejection sampling to create SFT data; report does not describe RL" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "Report does not claim distillation for the 405B; 405B-generated SFT data for 8B/70B is teacher-generated data, noted in prose not here" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Image/video/speech extensions described via adapters but explicitly 'still under development and not yet ready for release' (Abstract / §8) — released models are text-only" }, "attachment": { "provenance": "n/a", "ref": "§8 unreleased adapters" } }, "efficiency": { "kv_cache": { "value": "GQA: KV cache ∝ 8 KV heads (vs 128 query heads)", "provenance": "derived", "note": "from GQA (8 KV heads, §3.2)" }, "quantization": { "value": "FP8 inference for 405B: FFN matmuls with row-wise dynamic scaling, max 1200, first/last layers unquantized (§6.2)", "provenance": "stated", "ref": "§6.2" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "BF16 pipeline parallelism across 16 GPUs on two machines with micro-batching (§6.1)", "provenance": "stated", "ref": "§6.1" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (5-shot)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 87.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU-Pro (5-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "IFEval", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HumanEval", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 89.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MBPP EvalPlus", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GSM8K (8-shot, CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 96.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MATH (0-shot CoT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 73.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ARC-C", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 96.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 51.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MGSM", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 91.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } } ] }, "contributions": { "claimed": [ { "text": { "value": "A herd of three models (8B/70B/405B) released under the Llama 3 Community License", "provenance": "stated", "ref": "Abstract / §1" }, "ref": { "value": "Abstract / §1", "provenance": "stated", "ref": "Abstract / §1" } }, { "text": { "value": "Data quality and quantity scaling: 15.6T tokens with a ~50/25/17/8 mix", "provenance": "stated", "ref": "§1 / §3.1" }, "ref": { "value": "§1 / §3.1", "provenance": "stated", "ref": "§1 / §3.1" } }, { "text": { "value": "Staged continued pretraining from 8K to 128K context", "provenance": "stated", "ref": "§3.4.2" }, "ref": { "value": "§3.4.2", "provenance": "stated", "ref": "§3.4.2" } }, { "text": { "value": "Post-training recipe: SFT → rejection sampling → DPO with formatting-token masking", "provenance": "stated", "ref": "§4" }, "ref": { "value": "§4", "provenance": "stated", "ref": "§4" } } ], "assessment": { "value": "Structurally refines the Llama 3.1 record's architecture (same family, this record covers the base April-2024 releases; the report's results refer to the 3.1 models). GQA + RoPE-500K + 128K context were already in the atlas via Llama 3.1; nothing structurally novel within atlas scope, but this is the canonical source for the base herd.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "from block: all layers dense (absence rule)" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "from block: GQA uniform across layers, no restricted patterns" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data composition disclosed but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "405B reference variant" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a general-purpose foundation model with broad benchmark coverage (§1)" } }, "prose": { "design_overview": { "value": "Llama 3 is a dense decoder-only transformer: GQA with 8 KV heads, RoPE at base 500,000, SwiGLU, a 128K-token vocabulary built from tiktoken plus 28K added tokens, and staged continued pretraining from 8K to 128K context. The report leads with data scale (15.6T tokens, ~50/25/17/8 knowledge/reasoning/code/multilingual mix) and a six-round post-training pipeline of SFT, rejection sampling, and DPO with formatting-token masking. The 405B flagship is the largest dense model in the atlas and the reference variant; the 8B/70B siblings share the block with shallower, narrower stacks. Architecturally it is the same family as the existing meta-llama-3.1 record — the report's results in fact describe the Llama 3.1 models — so this record's value is as the base-herd source: the pretraining recipe, staged context extension, and the explicit release of all three sizes under the Llama 3 Community License.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "dense decoder-only Transformer (self-attention + FFN blocks)", "provenance": "inferred", "note": "authored by atlas analyst; report cites Vaswani et al. 2017" }, "origin": { "value": "Transformer (Vaswani et al. 2017)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "RoPE with base 500,000", "provenance": "stated", "ref": "§3.2" }, "origin": { "value": "RoFormer / Llama line (per report §3.2 context)", "provenance": "inferred", "note": "report states the value; lineage naming is analyst judgement" } } ], "influence_out": [ { "model_slug": { "value": "allenai-olmo-3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "RoPE base 500,000", "provenance": "inferred", "note": "mechanism as stated in allenai-olmo-3 report's lineage_in" } }, { "model_slug": { "value": "allenai-olmo-2-7b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "RoPE base frequency 500,000 (raised from 10,000)", "provenance": "inferred", "note": "mechanism as stated in allenai-olmo-2-7b report's lineage_in" } } ], "notable_omissions": { "value": [ "Normalization type and placement never named", "Embedding tying undisclosed", "Partial RoPE application undisclosed", "Head dimension not stated directly (derived here)", "Report's own note: results refer to Llama 3.1 models (§1) — base-vs-instruct distinction must be read carefully", "Source stored as PDF extraction (arxiv.org/html 404s for this paper; ar5iv carries a fatal-error banner) — flag if table parsing matters" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "meta-llama-3.json" }, { "slug": { "value": "microsoft-phi-3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.1", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2404.14219", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline (meta.json)" }, "stored_path": { "value": "data/sources/microsoft-phi-3", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Phi-3", "provenance": "stated", "ref": "Title / Abstract", "note": "Report titled 'Phi-3 Technical Report'; abstract introduces phi-3-mini, phi-3-small, phi-3-medium and the 'phi-3.5 series' (mini, MoE, Vision)." }, "variants": [ { "value": "phi-3-mini (3.8B)", "provenance": "stated", "ref": "Abstract / §1" }, { "value": "phi-3-mini-128K (3.8B)", "provenance": "stated", "ref": "§2", "note": "Long-context version of phi-3-mini: 'extends the context length to 128K, called phi-3-mini-128K'." }, { "value": "phi-3-small (7B)", "provenance": "stated", "ref": "Abstract / §2" }, { "value": "phi-3-medium (14B)", "provenance": "stated", "ref": "Abstract / §2 Training Methodology" }, { "value": "phi-3.5-mini (3.8B)", "provenance": "stated", "ref": "Abstract / §4" }, { "value": "phi-3.5-MoE (16x3.8B)", "provenance": "stated", "ref": "Abstract / §2" }, { "value": "phi-3.5-Vision (4.2B)", "provenance": "stated", "ref": "Abstract / §7" } ], "org": { "value": "Microsoft", "provenance": "stated", "ref": "Title block" }, "release_date": { "value": "2024-04-22", "provenance": "inferred", "note": "arXiv v1 submission date (ID from report URL; arXiv API metadata) — report text itself is undated" }, "report_url": { "value": "https://arxiv.org/abs/2404.14219", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "Report names no license." }, "open_weights": { "value": null, "provenance": "unknown", "note": "Report never explicitly states that weights are released; the 'open source community' framing (§2) expresses intent, not a release statement." } }, "scale": { "total_params": { "value": 3.8, "provenance": "stated", "ref": "Abstract / §1 / §2", "note": "Reference variant phi-3-mini: 'a 3.8 billion parameter language model'." }, "active_params_per_token": { "value": 3.8, "provenance": "derived", "note": "Dense model: active params per token = total params (3.8B); no routing in phi-3-mini (MoE described only for the phi-3.5-MoE variant, §2)." }, "layers": { "value": 32, "provenance": "stated", "ref": "§2" }, "hidden_dim": { "value": 3072, "provenance": "stated", "ref": "§2" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Report does not disclose the FFN hidden dimension for any variant." }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "ffn_inner_dim not disclosed; ratio cannot be computed." }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "§2" }, "attention_heads_kv": { "value": 32, "provenance": "derived", "note": "MHA: KV heads = query heads = 32. MHA itself is inferred: report lists '32 heads' for phi-3-mini and describes grouped-query attention only for phi-3-small ('4 queries sharing 1 key', §2)." }, "head_dim": { "value": 96, "provenance": "derived", "note": "hidden_dim / attention_heads_q = 3072 / 32 = 96." }, "vocab_size": { "value": 32064, "provenance": "stated", "ref": "§2 (footnote 1)", "note": "Same tokenizer as Llama-2; BoS tokens removed and additional tokens added for the chat template." }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Report silent on input/output embedding tying." }, "variants": [ { "name": { "value": "phi-3-small", "provenance": "stated", "ref": "§2" }, "total_params": { "value": 7, "provenance": "stated", "ref": "Abstract / §2", "note": "'The phi-3-small model (7B parameters)'." }, "layers": { "value": 32, "provenance": "stated", "ref": "§2" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "§2" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed." }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "§2" }, "attention_heads_kv": { "value": 8, "provenance": "derived", "note": "GQA: '4 queries sharing 1 key' → 32 / 4 = 8 KV heads (§2)." }, "context_length": { "value": 8192, "provenance": "stated", "ref": "§2", "note": "'has default context length 8192'." } }, { "name": { "value": "phi-3-medium", "provenance": "stated", "ref": "Abstract / §2 Training Methodology" }, "total_params": { "value": 14, "provenance": "stated", "ref": "Abstract / §2 Training Methodology" }, "layers": { "value": 40, "provenance": "stated", "ref": "§2 Training Methodology", "note": "'The model has 40 heads and 40 layers'." }, "hidden_dim": { "value": 5120, "provenance": "stated", "ref": "§2 Training Methodology", "note": "'embedding dimension 5120'." }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "Not disclosed." }, "attention_heads_q": { "value": 40, "provenance": "stated", "ref": "§2 Training Methodology" }, "attention_heads_kv": { "value": 40, "provenance": "derived", "note": "Report: phi-3-medium uses 'the same tokenizer and architecture of phi-3-mini' (§2); mini is MHA → KV heads = query heads = 40." }, "context_length": { "value": null, "provenance": "unknown", "note": "Report does not state phi-3-medium's context length." } } ] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "Reference variant phi-3-mini is a fully dense transformer decoder (32 layers, no routing); MoE layers exist only in the phi-3.5-MoE variant (§2)." }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§2", "note": "phi-3-mini is a dense transformer decoder; MoE described only for phi-3.5-MoE (16 experts, top-2)." }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§2" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§2" }, "routing": { "value": null, "provenance": "n/a", "ref": "§2" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§2" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§2" } }, "attention_variant": { "value": "MHA", "provenance": "inferred", "note": "Report lists '32 heads' for phi-3-mini with no KV sharing and describes grouped-query attention only for phi-3-small ('4 queries sharing 1 key', §2); absence rule → MHA." }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "Report describes no per-layer attention variation for phi-3-mini; dense/blocksparse alternation is disclosed only for phi-3-small (§2)." }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "Report never names phi-3-mini's base positional encoding; it names only the LongRope extension (§2, §4). The stated 'similar block structure as Llama-2' implies RoPE but is not stated." }, "rope_base": { "value": null, "provenance": "unknown", "note": "Not disclosed." }, "partial_rope": { "value": null, "provenance": "unknown", "note": "Report silent on the dimensional scope of the (unnamed) positional encoding." }, "extension": { "method": { "value": "other", "provenance": "stated", "ref": "§2 / §4", "note": "LongRope (LongRoPE, cited [DZZ+24a]) extends phi-3-mini 4K to 128K; for the phi-3.5 series the report adds 'a mixed context window approach' (§4)." }, "factor": { "value": 32, "provenance": "derived", "note": "128K / 4K = 32 (deployed 131072 / trained 4096); scaling factor not stated directly." }, "target_length": { "value": 131072, "provenance": "derived", "note": "Report prints '128K' (§2, §4); stored as 128 x 1024 = 131072." } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "Report never names a normalization layer for any phi-3 variant." }, "placement": { "value": null, "provenance": "unknown", "note": "Report silent on pre/post placement of normalization." }, "qk_norm": { "value": null, "provenance": "unknown", "note": "Report silent on QK-normalisation; per-variant attention internals beyond head counts are not described." } }, "activation": { "value": null, "provenance": "unknown", "note": "Report names GEGLU only for phi-3-small ('We switched to GEGLU activation', §2); never names phi-3-mini's activation." }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "Report describes no designed sink tokens; any sink behaviour would be emergent (prose)." }, "softcapping": { "value": null, "provenance": "unknown", "note": "No logit softcapping disclosed." }, "other": { "value": null, "provenance": "unknown", "note": "No block-level stability mechanisms disclosed for phi-3-mini; phi-3-small uses muP for training stability (§2) — a training-side measure, not block-level." } }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: report describes its block in detail and never mentions attention over prior layer outputs; standard sequential residual connections are the default mechanism." } }, "context": { "trained_length": { "value": 4096, "provenance": "stated", "ref": "§2", "note": "Report: 'default context length 4K' for phi-3-mini." }, "deployed_length": { "value": 131072, "provenance": "derived", "note": "Report prints '128K' for the long-context version phi-3-mini-128K (§2); stored as 128 x 1024 = 131072. The default phi-3-mini serves 4K." }, "extension_method": { "value": "LongRope (LongRoPE, [DZZ+24a]); the phi-3.5 series additionally uses 'a mixed context window approach' during mid-training", "provenance": "stated", "ref": "§2 / §4" } }, "tokenizer": { "algorithm": { "value": "SentencePiece BPE (same tokenizer as Llama-2)", "provenance": "inferred", "note": "Report states phi-3-mini 'uses the same tokenizer with vocabulary size of 32064' as Llama-2 (§2) and never names an algorithm; it names tiktoken only for phi-3-small. Llama-2's tokenizer is a SentencePiece BPE (secondary knowledge used only for this inference)." }, "notes": { "value": "Same tokenizer as Llama-2: BoS tokens removed and additional tokens added for the chat template (footnote 1, §2); vocab size 32064 lives in scale.vocab_size. phi-3-small instead 'leverages the tiktoken tokenizer (for better multilingual tokenization) with a vocabulary size of 100352' with unused tokens removed (§2).", "provenance": "stated", "ref": "§2" } }, "training": { "tokens": { "value": 3.3, "provenance": "stated", "ref": "Abstract / §1 / §2", "note": "phi-3-mini 'trained on 3.3 trillion tokens'; 'We trained using bfloat16 for a total of 3.3T tokens'." }, "data_composition": { "value": "Heavily filtered publicly available web data (filtered to an 'educational level' from various open internet sources) plus synthetic LLM-generated data; mixture proportions not disclosed. phi-3-small adds 'an additional 10% multilingual data' (§2). phi-3-medium and phi-3-small trained on 4.8T tokens (Abstract).", "provenance": "stated", "ref": "Abstract / §1 / Training Methodology" }, "curriculum": { "value": "Two disjoint sequential pre-training phases: phase-1 mostly web sources for general knowledge and language understanding; phase-2 a more heavily filtered subset of the web data plus synthetic data for logical reasoning and niche skills. Framed as the 'data optimal regime' — calibrating data quality for a given scale — rather than the compute-optimal regime.", "provenance": "stated", "ref": "Training Methodology" }, "optimizer": { "value": null, "provenance": "unknown", "note": "Not disclosed." }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "Not disclosed (muP determines hyperparameters for phi-3-small, §2, but no schedule is given)." }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "Not disclosed." }, "precision": { "value": "bfloat16", "provenance": "stated", "ref": "§2" }, "parallelism": { "value": null, "provenance": "unknown", "note": "Not disclosed." }, "hardware": { "value": null, "provenance": "unknown", "note": "Not disclosed; the iPhone 14 deployment (Figure 2) is inference, not training hardware." }, "compute": { "value": null, "provenance": "unknown", "note": "Report discloses training tokens (3.3T) but no FLOPs or GPU-hours." } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "Post-training / §2", "note": "Two post-training stages: SFT then DPO. SFT uses highly curated data across math, coding, reasoning, conversation, model identity and safety, starting with English-only examples; phi-3-mini 'is already chat-finetuned' (§2)." }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "Post-training", "note": "DPO data covers chat format, reasoning and responsible-AI (RAI) efforts; outputs are used as 'rejected' responses to steer the model away from unwanted behaviour." }, "reasoning_training": { "value": null, "provenance": "unknown", "note": "Report describes SFT data covering reasoning domains but discloses no dedicated CoT-SFT, RLVR, or test-time reasoning training." }, "distillation": { "value": null, "provenance": "unknown", "note": "Report mentions no distillation from a teacher model; synthetic pre-training data is LLM-generated, which per schema §8 is teacher-generated data, not distillation." } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Reference variant phi-3-mini is described purely as a language model with text benchmarks (§3); the report's multimodal model is the separate phi-3.5-Vision (CLIP ViT-L/14 encoder + phi-3.5-mini decoder, §7.1)." }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§2", "note": "Text-only reference variant; no non-text modality attaches. (phi-3.5-Vision attaches a CLIP ViT-L/14 image encoder, §7.1.)" } }, "efficiency": { "kv_cache": { "value": "MHA with 32 heads: full per-head KV cache with no sharing, so KV-cache size scales with all 32 heads", "provenance": "derived", "note": "Consequence of attention_variant MHA. The report discusses KV-cache reduction only for phi-3-small's blocksparse attention ('significant KV cache reduction', §2)." }, "quantization": { "value": "4-bit quantization: phi-3-mini occupies approximately 1.8 GB and runs natively on an iPhone 14 (A16 Bionic), fully offline, at over 12 tokens per second", "provenance": "stated", "ref": "§2 / Figure 2" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "Report does not mention speculative decoding, draft models, or multi-token prediction." }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "No serving optimisations disclosed for phi-3-mini; custom Triton (Flash Attention based) training kernels and vLLM paged-attention inference kernels are described only for phi-3-small's blocksparse attention (§2)." } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (5-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 68.8, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "MT Bench (2 round ave.)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 8.38, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "HellaSwag (5-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 76.7, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "GSM-8K (8-Shot; CoT)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 82.5, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "MATH (0-Shot; CoT)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 41.3, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "MedQA (2-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 53.8, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "TriviaQA (5-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 64.0, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "Arc-C (10-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 84.9, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "BigBench-Hard (3-Shot; CoT)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 71.7, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "WinoGrande (5-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 70.8, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "TruthfulQA (10-Shot; MC2)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 65.0, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "BoolQ (2-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 77.2, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "HumanEval (0-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 58.5, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "MBPP (3-Shot)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 70.0, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } }, { "name": { "value": "GPQA (2-Shot; CoT)", "provenance": "stated", "ref": "§3 benchmark table" }, "value": { "value": 32.8, "provenance": "stated", "ref": "§3 benchmark table (phi-3-mini column)" }, "ref": { "value": "§3 benchmark table (phi-3-mini column)", "provenance": "stated", "ref": "§3 benchmark table" } } ] }, "contributions": { "claimed": [ { "text": { "value": "The 'data optimal regime': calibrating data quality (heavily filtered web plus synthetic data) for a given model scale rather than chasing compute-optimal scaling, so a 3.8B model rivals Mixtral 8x7B and GPT-3.5.", "provenance": "stated", "ref": "Abstract / §1 / Training Methodology" }, "ref": { "value": "Abstract / §1 / Training Methodology", "provenance": "stated", "ref": "Abstract / §1 / Training Methodology" } }, { "text": { "value": "A long-context version, phi-3-mini-128K, that extends the default 4K context to 128K via LongRope.", "provenance": "stated", "ref": "§2 / §4" }, "ref": { "value": "§2 / §4", "provenance": "stated", "ref": "§2 / §4" } }, { "text": { "value": "Blocksparse attention in phi-3-small: per-head sparsity patterns over the KV cache with dense and blocksparse layers alternated, backed by custom Triton (Flash Attention based) training kernels and a vLLM paged-attention inference kernel for real deployment speed-ups.", "provenance": "stated", "ref": "§2" }, "ref": { "value": "§2", "provenance": "stated", "ref": "§2" } }, { "text": { "value": "phi-3-small's recipe: grouped-query attention (4 queries per key), GEGLU activation, and Maximal Update Parametrization (muP) to tune hyperparameters on a proxy model and transfer them, for better performance and training stability.", "provenance": "stated", "ref": "§2" }, "ref": { "value": "§2", "provenance": "stated", "ref": "§2" } }, { "text": { "value": "On-device capability: 4-bit quantized phi-3-mini (about 1.8 GB) running natively and fully offline at over 12 tokens per second on an iPhone 14 with A16 Bionic.", "provenance": "stated", "ref": "§2 / Figure 2" }, "ref": { "value": "§2 / Figure 2", "provenance": "stated", "ref": "§2 / Figure 2" } }, { "text": { "value": "Two-stage post-training (SFT + DPO) with responsible-AI alignment: red-teaming, automated testing, and safety evaluation across dozens of RAI harm categories, substantially reducing harmful response rates.", "provenance": "stated", "ref": "Post-training / §5" }, "ref": { "value": "Post-training / §5", "provenance": "stated", "ref": "Post-training / §5" } } ], "assessment": { "value": "The genuinely novel pieces are the 'data-optimal regime' training recipe — heavily filtered web plus synthetic data at small scale, explicitly deviating from compute-optimal scaling laws — and phi-3-small's blocksparse attention with custom kernels; nearly everything else (Llama-2-style decoder block, MHA, GQA, GEGLU, muP, LongRope extension, SFT + DPO) is adopted from prior architectures the report itself names and cites. The headline claim that a 3.8B model rivals Mixtral 8x7B and GPT-3.5 rests on the report's own evaluation pipeline, which the report discloses as unoptimised for phi-3 — a credible but self-run comparison. The phi-3.5-MoE variant is a conventional top-2-of-16 routed MoE whose SparseMixer router training is cited from prior work, adding little architectural novelty. Relative to other atlas entries, phi-3 is architecturally conservative (like Llama 3.1) but methodologically distinctive in making data quality, not compute, the primary lever — and it is the atlas taxonomy's anchor for on-device design intent.", "provenance": "inferred", "note": "authored by atlas analyst (comparison against other atlas entries)" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "Reference variant phi-3-mini is fully dense (no routing in any of its 32 layers); MoE appears only in the phi-3.5-MoE variant (§2)." }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "phi-3-mini uses unrestricted full attention in all layers; restricted per-head sparsity (blocksparse) appears only in phi-3-small (§2)." }, "modality": { "value": "text-only", "provenance": "derived", "note": "From modality.type: reference variant is text-only; the multimodal phi-3.5-Vision is a separate variant (§7)." }, "openness": { "value": "undisclosed", "provenance": "derived", "note": "Report never states weights are released (open_weights unknown) and discloses no release of training data; the 'open source community' framing (§2) is not a release statement." }, "scale_class": { "value": "medium", "provenance": "derived", "note": "Reference variant 3.8B: 1B <= 3.8B < 10B." }, "design_intent": { "value": "on-device", "provenance": "inferred", "note": "Report framing leads with on-device deployment: title 'A Highly Capable Language Model Locally on Your Phone', abstract 'small enough to be deployed on a phone', and an iPhone 14 demo at over 12 tokens/s (§2); broad benchmark coverage is secondary framing." } }, "prose": { "design_overview": { "value": "Phi-3-mini is a compact dense decoder-only transformer built, per the report, on a block structure similar to Llama-2: 32 layers, hidden size 3072, 32 attention heads, and a 32,064-token vocabulary shared with the Llama-2 tokenizer (BoS tokens removed, chat tokens added). The report never names its positional encoding, normalization, or activation — only dimensions and head counts — though the stated Llama-2 lineage and the LongRope extension point to RoPE. Its defining contribution is not the block but the data: a 'data-optimal regime' in which heavily filtered web data (selected for 'educational level') and LLM-generated synthetic data are calibrated to a small model's capacity, trained in two sequential phases (web-heavy, then filtered-web plus synthetic) for 3.3T tokens in bfloat16. Default context is 4K, extended to 128K by LongRope for the phi-3-mini-128K variant. Post-training is a two-stage SFT + DPO pipeline with red-teaming and RAI safety evaluation. The family scales the recipe: phi-3-small (7B) swaps in the tiktoken tokenizer, GEGLU, muP-tuned hyperparameters, GQA (4 queries per key), and a novel blocksparse attention that alternates dense and sparse layers with custom Triton/vLLM kernels; phi-3-medium (14B) reuses mini's architecture at 40 layers and hidden size 5120. Later phi-3.5 variants add a top-2-of-16 MoE (6.6B active of 42B), long-context mid-training, and a CLIP-based vision variant. The result is a 3.8B model that rivals far larger models on the report's own benchmarks and runs fully offline on a phone.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Llama-2-style decoder block structure", "provenance": "inferred", "note": "authored by atlas analyst; report: 'phi-3-mini is built upon a similar block structure as Llama-2' (§2)" }, "origin": { "value": "Llama-2", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "same tokenizer as Llama-2 (vocab 32064; BoS removed, chat-template tokens added)", "provenance": "inferred", "note": "authored by atlas analyst; report states tokenizer identity (§2)" }, "origin": { "value": "Llama-2", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "LongRope context extension (4K to 128K)", "provenance": "inferred", "note": "authored by atlas analyst; report names LongRope, cited [DZZ+24a] (§2, §4)" }, "origin": { "value": "LongRoPE [DZZ+24a]", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "data-centric training recipe (filtered web + synthetic data) for small models", "provenance": "inferred", "note": "authored by atlas analyst; report: 'We follow the sequence of works initiated in Textbooks Are All You Need [GZA+23]' (Training Methodology)" }, "origin": { "value": "'Textbooks Are All You Need' / phi-1.5–phi-2 line [GZA+23, LBE+23, JBA+23]", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "grouped-query attention (4 queries per key) — phi-3-small", "provenance": "inferred", "note": "authored by atlas analyst; mechanism named in report (§2), origin not cited in report" }, "origin": { "value": "GQA (prior literature; not cited in report)", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "Maximal Update Parametrization (muP) hyperparameter transfer — phi-3-small", "provenance": "inferred", "note": "authored by atlas analyst; report names muP, cited [YHB+22] (§2)" }, "origin": { "value": "Tensor Programs V / muP [YHB+22]", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "Flash Attention based Triton training kernel and vLLM paged-attention inference kernel — phi-3-small", "provenance": "inferred", "note": "authored by atlas analyst; report cites [DFE+22] and [KLZ+23] (§2)" }, "origin": { "value": "Flash Attention [DFE+22] and vLLM [KLZ+23]", "provenance": "inferred", "note": "authored by atlas analyst" } }, { "mechanism": { "value": "SparseMixer sparse-backpropagation router training — phi-3.5-MoE", "provenance": "inferred", "note": "authored by atlas analyst; report names SparseMixer, cited [LGC23, LDL+23] (§2)" }, "origin": { "value": "SparseMixer [LGC23, LDL+23]", "provenance": "inferred", "note": "authored by atlas analyst" } } ], "influence_out": [ { "model_slug": { "value": "microsoft-phi-4", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "full attention, tiktoken tokenizer, midtraining", "provenance": "inferred", "note": "mechanism as stated in microsoft-phi-4 report's lineage_in" } } ], "notable_omissions": { "value": [ "No release statement: the report never explicitly says the weights are released (openness classified undisclosed) and names no license.", "Base positional encoding, normalization type/placement, and activation are never named for phi-3-mini (implied only by the stated Llama-2-like block).", "FFN hidden dimension not disclosed for any variant, so ffn_ratio is not computable.", "Training details peers disclose are absent: optimizer, learning-rate and batch schedules, hardware, and FLOPs/GPU-hours.", "Data mixture proportions not disclosed — composition is qualitative, and the two-phase split carries no token budgets.", "No long-context evaluation of phi-3-mini-128K itself: RULER and RepoQA results cover only the phi-3.5 models (§4), which the report admits drop sharply at the 128K window.", "The 'mixed context window approach' used for the phi-3.5 series' long-context mid-training is not described.", "Multilingual coverage acknowledged as a weakness for phi-3-mini (mostly English); multilingual results exist only for phi-3-small and the phi-3.5 series." ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "microsoft-phi-3.json" }, { "slug": { "value": "microsoft-phi-4", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2412.08905", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/microsoft-phi-4", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Phi-4", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "Microsoft Research", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2024-12-12", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date per export.arxiv.org API (arXiv:2412.08905)" }, "report_url": { "value": "https://arxiv.org/abs/2412.08905", "provenance": "stated", "ref": "pipeline" }, "license": { "provenance": "unknown" }, "open_weights": { "provenance": "unknown" } }, "scale": { "total_params": { "value": 14, "provenance": "stated", "ref": "Abstract / §1 / Table 1" }, "active_params_per_token": { "value": 14, "provenance": "derived", "note": "Dense model: active = total (14B)" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "ffn_ratio": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "head_dim": { "provenance": "unknown" }, "vocab_size": { "value": 100352, "provenance": "stated", "ref": "§3 (padded vocabulary size of 100,352 including unused tokens)" }, "embedding_tied": { "provenance": "unknown" }, "variants": [] }, "block": { "block_type": { "value": "dense", "provenance": "inferred", "note": "Absence rule: 'decoder-only transformer' with no routing described (§3)" }, "moe": { "expert_count": { "provenance": "n/a", "ref": "§3 dense block" }, "experts_per_token": { "provenance": "n/a", "ref": "§3 dense block" }, "shared_experts": { "provenance": "n/a", "ref": "§3 dense block" }, "routing": { "provenance": "n/a", "ref": "§3 dense block" }, "load_balancing": { "provenance": "n/a", "ref": "§3 dense block" }, "expert_granularity": { "provenance": "n/a", "ref": "§3 dense block" } }, "attention_variant": { "value": "MHA", "provenance": "inferred", "note": "Report states full attention over the 4K context 'rather than a 2K sliding window used in phi-3-medium' and that the model 'closely follows phi-3-medium' (§3); no KV grouping disclosed" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "Absence rule: full attention over the entire 4K context (§3)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule" }, "position_encoding": { "method": { "provenance": "unknown" }, "rope_base": { "provenance": "unknown" }, "partial_rope": { "provenance": "unknown" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "Context extended to 16K during midtraining (§3)", "ref": "§3" }, "factor": { "provenance": "unknown" }, "target_length": { "value": 16384, "provenance": "stated", "ref": "§3" } } }, "normalization": { "type": { "provenance": "unknown" }, "placement": { "provenance": "unknown" }, "qk_norm": { "provenance": "unknown", "note": "Report's architecture description is not detailed enough for the absence rule" } }, "activation": { "provenance": "unknown" }, "stability": { "attention_sinks": { "provenance": "unknown", "note": "Report not detailed enough for the absence rule" }, "softcapping": { "provenance": "unknown", "note": "Report not detailed enough for the absence rule" }, "other": { "provenance": "unknown", "note": "Report not detailed enough for the absence rule" } } }, "context": { "trained_length": { "value": 4096, "provenance": "stated", "ref": "§3" }, "deployed_length": { "value": 16384, "provenance": "stated", "ref": "§3 (extended to a 16K context length during midtraining)" }, "extension_method": { "value": "extended to 16K during midtraining", "provenance": "stated", "ref": "§3" } }, "tokenizer": { "algorithm": { "value": "tiktoken-based BPE", "provenance": "stated", "ref": "§3" }, "notes": { "value": "Padded vocabulary of 100,352 tokens including unused tokens (§3)", "provenance": "stated", "ref": "§3" } }, "training": { "tokens": { "value": 10, "provenance": "stated", "ref": "§3 (pretrained for approximately 10T tokens)" }, "data_composition": { "value": "Synthetic-first: ~50 broad types of synthetic datasets (~400B unweighted tokens) for pretraining/midtraining; curated and filtered organic data (arXiv, PubMed Central, GitHub, licensed books, filtered web); multilingual coverage (DE/ES/FR/PT/IT/HI/JA) (§2)", "provenance": "stated", "ref": "§2" }, "curriculum": { "value": "Pretraining → midtraining (16K context stage)", "provenance": "stated", "ref": "§3" }, "optimizer": { "provenance": "unknown" }, "lr_schedule": { "value": "Linear warm-up and decay; peak learning rate 0.0003; constant weight decay 0.1", "provenance": "stated", "ref": "§3" }, "batch_schedule": { "value": "Global batch size 5760", "provenance": "stated", "ref": "§3" }, "precision": { "provenance": "unknown" }, "parallelism": { "provenance": "unknown" }, "hardware": { "provenance": "unknown" }, "compute": { "provenance": "unknown" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1 (one round of SFT)" }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "§4.2 (two rounds of DPO: first with pivotal-token-search pairs, second with full-length preference pairs)" }, "reasoning_training": { "value": "none disclosed", "provenance": "inferred", "note": "Report discusses long-CoT models (o1, R1-Lite) as context but describes no reasoning RL for phi-4" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "Report claims phi-4 'goes beyond distillation' (§1); GPT-4o-generated training data is teacher-generated data, not distillation (schema note)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes text tasks only" }, "attachment": { "provenance": "n/a", "ref": "§2 model description" } }, "efficiency": { "kv_cache": { "value": "Full attention with no grouping disclosed: KV cache per token scales with all attention heads; 16K deployed context multiplies cache", "provenance": "derived", "note": "from attention_variant (inferred MHA, §3)" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 84.8, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "GPQA", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 56.1, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "MATH", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 80.4, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "HumanEval", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 82.6, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "MGSM", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 80.6, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "SimpleQA", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 3.0, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "DROP", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 75.5, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 70.4, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "HumanEval+", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 82.8, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "ArenaHard", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 75.4, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "IFEval", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 63.0, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "PhiBench", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 56.2, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Midtraining: an intermediate training stage between pretraining and post-training", "provenance": "stated", "ref": "§1 / §2.2 / §3" }, "ref": { "value": "§1 / §2.2 / §3", "provenance": "stated", "ref": "§1 / §2.2 / §3" } }, { "text": { "value": "Pivotal Token Search (PTS): a technique to create DPO pairs from pivotal tokens", "provenance": "stated", "ref": "§2.4 / §4.3" }, "ref": { "value": "§2.4 / §4.3", "provenance": "stated", "ref": "§2.4 / §4.3" } }, { "text": { "value": "Synthetic-first data: 50 broad types of synthetic datasets with novel generation methodologies", "provenance": "stated", "ref": "§1 / §2.2" }, "ref": { "value": "§1 / §2.2", "provenance": "stated", "ref": "§1 / §2.2" } }, { "text": { "value": "Going 'beyond distillation': matches or exceeds much larger models (e.g. Llama-3.1-405B) on reasoning benchmarks, surpassing its teacher GPT-4o on STEM QA", "provenance": "stated", "ref": "Abstract / §1" }, "ref": { "value": "Abstract / §1", "provenance": "stated", "ref": "Abstract / §1" } } ], "assessment": { "value": "Within the atlas, phi-4 is a direct refinement of the existing microsoft-phi-3 line: same 'small model, dense, data-centric' intent, with the 2K sliding window dropped for full attention and a tiktoken vocabulary. The architectural novelty is modest; the claimed novelty is in the data and post-training recipes (midtraining, PTS), which the schema records under training/post_training rather than block design.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "from block: dense decoder-only (absence rule)" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "from block: full attention over 4K context (§3)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "undisclosed", "provenance": "derived", "note": "report silent on weight release and license" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "14B reference variant" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "report leads with STEM/reasoning framing: 'substantially surpasses its teacher on STEM-focused QA' (Abstract, §1)" } }, "prose": { "design_overview": { "value": "Phi-4 is a 14B dense decoder-only transformer that 'closely follows phi-3-medium' with three stated deltas: a tiktoken tokenizer with a padded 100,352-token vocabulary, full attention over the 4K default context instead of phi-3-medium's 2K sliding window, and a midtraining stage that extends the context to 16K. The report discloses almost nothing about the block internals — no layer count, hidden dimension, head counts, normalization, or activation — so the record is deliberately sparse on scale beyond the headline 14B. The report's contribution is the recipe: ~10T tokens with a synthetic-first mixture (~400B unweighted synthetic tokens across 50 dataset types), curated organic data, then one round of SFT and two rounds of DPO, the first built on the novel Pivotal Token Search technique. The framing is explicitly about matching much larger models (Llama-3.1-405B) on reasoning benchmarks with a 14B parameter budget.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "full attention, tiktoken tokenizer, midtraining", "provenance": "stated", "ref": "§3" }, "origin": { "value": "Phi-3-medium (report: 'closely follows phi-3-medium, except…')", "provenance": "stated", "ref": "§3" } } ], "influence_out": [], "notable_omissions": { "value": [ "No layer count, hidden dimension, FFN dimension, or attention head counts disclosed", "No normalization, activation, or positional-encoding disclosure", "No optimizer name or numeric precision for pretraining", "Silent on weight release and license", "Benchmarks shown only as Table 1 comparisons without harness details" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "microsoft-phi-4.json" }, { "slug": { "value": "minimax-m2", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2605.26494", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/minimax-m2/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "MiniMax-M2 series", "provenance": "stated", "ref": "Title ('The MiniMax-M2 Series: Mini Activations Unleashing Max Real-World Intelligence')" }, "variants": [ { "value": "M2", "provenance": "stated", "ref": "Abstract ('The flagship M2 contains 229.9B total parameters')" }, { "value": "M2.5", "provenance": "stated", "ref": "§7.4 ('The previous public M2 release (M2.5) is included as the within-series reference')" }, { "value": "M2.7", "provenance": "stated", "ref": "Abstract ('the latest M2.7 checkpoint'), Figure 1 caption" } ], "org": { "value": "MiniMax", "provenance": "stated", "ref": "Title page byline (model@minimax.io)" }, "release_date": { "value": "2026-05-26", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date 2026-05-26 per export.arxiv.org API metadata (arXiv:2605.26494; v2 2026-07-30)" }, "report_url": { "value": "https://arxiv.org/abs/2605.26494", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "the arXiv.org perpetual non-exclusive license printed on the page covers the paper, not the model weights; no model license named" }, "open_weights": { "value": true, "provenance": "stated", "ref": "§7.4 ('The previous public M2 release (M2.5)'); §1 (within-series checkpoints released publicly; M2.7 is the current checkpoint)" } }, "scale": { "total_params": { "value": 229.9, "provenance": "stated", "ref": "§2.1 ('It contains 229.9B total parameters, with 9.8B activated per token')" }, "active_params_per_token": { "value": 9.8, "provenance": "stated", "ref": "§2.1" }, "layers": { "value": 62, "provenance": "stated", "ref": "§2.1 ('a 62-layer decoder-only Transformer')" }, "hidden_dim": { "value": 3072, "provenance": "stated", "ref": "§2.1 ('a hidden dimension of 3,072')" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "per-expert FFN size not disclosed; only that fine-grained experts reduce per-expert FFN size (§2.2.1)" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "ffn_inner_dim unknown, so the ratio cannot be derived" }, "attention_heads_q": { "value": 48, "provenance": "stated", "ref": "§2.1 ('using 48 query heads and 8 key-value heads (GQA)')" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "§2.1" }, "head_dim": { "value": 64, "provenance": "derived", "note": "3072 / 48 = 64 (hidden_dim / attention_heads_q); report does not state head dimension directly" }, "vocab_size": { "value": 200064, "provenance": "stated", "ref": "§2.1 ('a vocabulary size of 200,064')" }, "embedding_tied": { "value": null, "provenance": "unknown" }, "variants": [ { "name": { "value": "M2.5", "provenance": "stated", "ref": "§7.4 ('The previous public M2 release (M2.5)')" }, "total_params": { "value": null, "provenance": "unknown", "note": "no per-variant total disclosed; the report frames M2/M2.5/M2.7 as checkpoints of one architecture ('components that co-evolve from M2 through M2.5 to the latest M2.7', §1)" }, "layers": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "not separately disclosed" } }, { "name": { "value": "M2.7", "provenance": "stated", "ref": "Abstract ('the latest M2.7 checkpoint'), Figure 1 caption" }, "total_params": { "value": null, "provenance": "unknown", "note": "no per-variant total disclosed; Figure 1 caption states 'only ~10B activated parameters' for M2.7, matching the M2 flagship's 9.8B active footprint" }, "layers": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not separately disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "not separately disclosed" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "every Transformer block pairs self-attention with an MoE feed-forward layer (§2.1); 62/62 layers routed MoE (256 experts, 8 activated)" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "§2.1 ('The MoE feed-forward layer contains 256 fine-grained experts')" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§2.1 ('with 8 experts activated per token')" }, "shared_experts": { "value": 0, "provenance": "inferred", "note": "absence rule: the MoE layer is described solely as 256 fine-grained routed experts (§2.2.1); no shared/always-active expert is ever mentioned" }, "routing": { "value": "sigmoid gating with learnable expert-specific bias terms and top-8 selection (no softmax zero-sum constraint)", "provenance": "stated", "ref": "§2.1, §2.2.1 ('Routing is implemented using sigmoid gating with learnable expert-specific bias terms'); 'Instead of softmax-based top-k gating... we use sigmoid gating'" }, "load_balancing": { "value": "learnable per-expert bias terms in the gating function, optimized jointly with model parameters, implicitly regulating expert utilization; auxiliary load-balancing loss greatly reduced (not eliminated)", "provenance": "stated", "ref": "§2.2.1 ('Expert Bias... implicitly regulate expert utilization, allowing the auxiliary load-balancing loss to be greatly reduced')" }, "expert_granularity": { "value": "fine-grained: larger number of smaller experts with reduced per-expert FFN size (following DeepSeekMoE)", "provenance": "stated", "ref": "§2.2.1 ('Fine-Grained Experts... a larger number of smaller experts, increasing the total expert count while reducing per-expert FFN size')" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2.1 ('full multi-head attention across all layers, using 48 query heads and 8 key-value heads (GQA)')" }, "attention_layer_pattern": { "value": "uniform — full attention in all 62 layers (no hybrid, no sliding window in the final design)", "provenance": "stated", "ref": "§2.1, §2.2.2 ('M2 adopts full multi-head attention across all layers'; hybrid SWA variants were explored and rejected)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: standard residual Transformer described ('each Transformer block... followed by an MoE feed-forward layer'); no attention-over-layers or learned connection matrix described" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1 ('Rotary Position Embeddings (RoPE) are applied throughout the model')" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE base frequency not disclosed" }, "partial_rope": { "value": false, "provenance": "inferred", "note": "absence rule: RoPE is applied 'throughout the model' with no subset-of-dims qualification" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "context reached by staged training at the deployed length — 'progressively extend the model's context window from 8K tokens through 32K and ultimately to 192K tokens' (§2.4); no RoPE interpolation, NTK, or YaRN described" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.4", "note": "no rescaling factor applies; no PE extension method described" }, "target_length": { "value": 192000, "provenance": "stated", "ref": "§2.1 ('a 192K-token native context window')" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "no normalization layer type named anywhere in the report (searched RMSNorm/LayerNorm)" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "QK-normalisation not described; absence rule not applied because the block is not described in full detail (no norm/activation disclosure)" } }, "activation": { "value": null, "provenance": "unknown", "note": "activation function not named anywhere in the report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: final design uses full attention with no designed sink tokens; sink tokens were added only in the rejected hybrid-SWA experiment variants (§2.2.2)" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no attention-logit softcapping described" }, "other": [ { "value": "Learnable per-expert bias terms in the sigmoid gating function regulate expert utilization and allow the auxiliary load-balancing loss to be greatly reduced", "provenance": "stated", "ref": "§2.2.1" }, { "value": "MTP loss weight annealing: initial weight 0.3 annealed to 0.1 during the decay phase of continued pre-training", "provenance": "stated", "ref": "§2.3" }, { "value": "MTP modules copy-initialized from the main model when expanding from one (K=1) to three (K=3) modules, avoiding the high-loss random-init transient that would temporarily degrade the main model", "provenance": "stated", "ref": "§2.3" } ] } }, "context": { "trained_length": { "value": 192000, "provenance": "stated", "ref": "§2.1 ('pre-trained on 29.2T tokens with a maximum context length of 192K'); §2.4 (8K → 32K → 192K staged extension)" }, "deployed_length": { "value": 192000, "provenance": "stated", "ref": "§2.1 ('a 192K-token native context window')" }, "extension_method": { "value": "multi-stage training curriculum: context window progressively extended from 8K through 32K to 192K during continued pre-training; no positional-encoding modification described", "provenance": "stated", "ref": "§2.4 ('Long-Context Extension... progressively extend the model's context window from 8K tokens through 32K and ultimately to 192K tokens')" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer never described; only the 200,064 vocab size appears (§2.1)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 29.2, "provenance": "stated", "ref": "§2.1 ('pre-trained on 29.2T tokens'); §2.4 (19.9T constant phase + 9.3T decay phase)" }, "data_composition": { "value": "comprehensive curated corpus: web documents, academic literature, books, programming code, and structured question-answering content; quality assessed by model-based reward scoring plus auxiliary classifiers; balanced sampling upweights high-quality content while retaining category diversity; code, mathematics, and STEM significantly upsampled relative to their natural distribution", "provenance": "stated", "ref": "§2.4 ('Training Data', 'Data Distribution')" }, "curriculum": { "value": "two-phase pre-training: constant phase on 19.9T tokens, then a decay phase with a 9.3T budget mixing short-text decay data with long-context data (high-quality code concatenation, naturally long-form PDFs, thematically related document packing); progressive context extension 8K → 32K → 192K; MTP loss weight annealed 0.3 → 0.1", "provenance": "stated", "ref": "§2.4, §2.3" }, "optimizer": { "value": null, "provenance": "unknown", "note": "optimizer not disclosed" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "learning-rate schedule not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "batch schedule not disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "no TP/PP/EP/CP strategy disclosed for pre-training (the report notes only that fine-grained experts reduce variance in expert utilization across devices, §2.2.1)" }, "hardware": { "value": null, "provenance": "unknown", "note": "no training hardware named (only generic 'GPU compute scaling' discussion, §2.2.2)" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§5 ('We conduct Supervised Fine-Tuning (SFT) to instill the desired interleaved thinking behavior in M2'); SFT data interleaves thinking traces with intermediate actions and observations" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§6.2.1 ('CISPO. We adapt Clipped Importance Sampling Policy Optimization (CISPO) [MiniMax-M1] to M2 series RL training'); clipped importance ratios with stop-gradient, zero lower bound, 1+ε upper bound" }, "reasoning_training": { "value": "agentic RL over MDP-formulated trajectories (LLM as policy, context management/tool execution/memory as environment): CISPO policy gradient with composite rewards — process reward (language-mixing and tool-format penalties, structured-reasoning rewards), outcome reward, and time-efficiency shaping; interleaved chain-of-thought with reasoning-state persistence (full thinking blocks appended to message history across turns); multi-stage context-length curriculum in RL", "provenance": "stated", "ref": "§6.2, §5, §7.1" }, "distillation": { "value": null, "provenance": "unknown", "note": "no distillation mentioned; the series checkpoints evolve through continued data/RL pipelines, not teacher distillation" } }, "modality": { "type": { "value": "multimodal", "provenance": "inferred", "note": "evaluation includes 'MM Claw, our internal multi-modal office-claw benchmark' (§7.3), implying the model processes non-text content; the report itself never describes a vision encoder or any modality-attachment mechanism" }, "attachment": { "value": null, "provenance": "unknown", "note": "no vision encoder or modality-attachment mechanism described anywhere in the report" } }, "efficiency": { "kv_cache": { "value": "GQA with 8 KV heads across all 62 full-attention layers: KV cache ∝ 8 heads and grows linearly with sequence length (no linear-attention layers to bound the cache)", "provenance": "derived", "note": "from attention_variant (GQA, 48 Q / 8 KV heads) and attention_layer_pattern (full attention in all layers), §2.1" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "no quantization format described" }, "speculative_dedup": { "value": "MTP-based speculative decoding: three MTP modules (K=3) generate draft tokens verified by the main model in a single forward pass; MTP modules co-trained with the RL policy via a top-K KL divergence loss to keep acceptance rates high under distribution shift; copy-initialized from main-model weights", "provenance": "stated", "ref": "§2.3, §6.3 ('MTP-based Speculative Decoding')" }, "serving": { "value": "heterogeneous prefill-decode disaggregation with per-phase parallelism strategies optimized for MoE (eliminating prefill/decode mutual interference); inference kernels co-designed with the deployment stack inside the Forge RL system", "provenance": "stated", "ref": "§6.3 ('Heterogeneous Prefill-Decode Disaggregation'), §1 (Forge)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "SWE-bench Pro", "provenance": "stated", "ref": "Abstract, §7.5" }, "value": { "value": 56.2, "provenance": "stated", "ref": "Abstract, §7.5" }, "ref": { "value": "Abstract, §7.5", "provenance": "stated", "ref": "Abstract, §7.5" } }, { "name": { "value": "SWE-bench Multilingual", "provenance": "stated", "ref": "Abstract, §7.5" }, "value": { "value": 76.5, "provenance": "stated", "ref": "Abstract, §7.5" }, "ref": { "value": "Abstract, §7.5", "provenance": "stated", "ref": "Abstract, §7.5" } }, { "name": { "value": "Multi-SWE-bench", "provenance": "stated", "ref": "Abstract, §7.5" }, "value": { "value": 52.7, "provenance": "stated", "ref": "Abstract, §7.5" }, "ref": { "value": "Abstract, §7.5", "provenance": "stated", "ref": "Abstract, §7.5" } }, { "name": { "value": "Terminal-Bench 2.0", "provenance": "stated", "ref": "Abstract, §7.5" }, "value": { "value": 57.0, "provenance": "stated", "ref": "Abstract, §7.5" }, "ref": { "value": "Abstract, §7.5", "provenance": "stated", "ref": "Abstract, §7.5" } }, { "name": { "value": "NL2Repo", "provenance": "stated", "ref": "§7.5" }, "value": { "value": 39.8, "provenance": "stated", "ref": "§7.5" }, "ref": { "value": "§7.5", "provenance": "stated", "ref": "§7.5" } }, { "name": { "value": "MLE Bench Lite (medal rate %)", "provenance": "stated", "ref": "§7.5" }, "value": { "value": 66.6, "provenance": "stated", "ref": "§7.5" }, "ref": { "value": "§7.5", "provenance": "stated", "ref": "§7.5" } }, { "name": { "value": "VIBE-Pro", "provenance": "stated", "ref": "§7.5" }, "value": { "value": 55.6, "provenance": "stated", "ref": "§7.5" }, "ref": { "value": "§7.5", "provenance": "stated", "ref": "§7.5" } }, { "name": { "value": "HyperTask", "provenance": "stated", "ref": "§7.5" }, "value": { "value": 67.6, "provenance": "stated", "ref": "§7.5" }, "ref": { "value": "§7.5", "provenance": "stated", "ref": "§7.5" } }, { "name": { "value": "MM Claw", "provenance": "stated", "ref": "Abstract, §7.3" }, "value": { "value": 62.7, "provenance": "stated", "ref": "Abstract, §7.3" }, "ref": { "value": "Abstract, §7.3", "provenance": "stated", "ref": "Abstract, §7.3" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Abstract" }, "value": { "value": 77.8, "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "name": { "value": "GDPval-AA", "provenance": "stated", "ref": "Abstract" }, "value": { "value": 50.0, "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "name": { "value": "Toolathlon", "provenance": "stated", "ref": "Abstract" }, "value": { "value": 46.3, "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "name": { "value": "AIME 2026", "provenance": "stated", "ref": "Abstract" }, "value": { "value": 94.2, "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "Abstract" }, "value": { "value": 89.8, "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } } ] }, "contributions": { "claimed": [ { "text": { "value": "The MiniMax-M2 series: a 229.9B-total / 9.8B-active MoE family showing that 'mini activations can unleash maximum real-world intelligence' — frontier-tier agentic performance at ~1/20 of the activated footprint of closed frontier systems", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } }, { "text": { "value": "Agent-driven data pipelines producing large-scale verifiable trajectories across agentic coding (SWE, AppDev, Terminal-Gym) and agentic cowork, each grounded in an executable workspace with artifact-aligned rewards (Agent-as-a-Verifier for AppDev)", "provenance": "stated", "ref": "§1, §4" }, "ref": { "value": "§1, §4", "provenance": "stated", "ref": "§1, §4" } }, { "text": { "value": "Forge, an agent-native RL system: windowed-FIFO scheduling absorbing trajectory-length variance, prefix-tree merging with up to 40× training speedup, and clean training–inference–agent decoupling supporting both white-box and black-box agents", "provenance": "stated", "ref": "§1, §6" }, "ref": { "value": "§1, §6", "provenance": "stated", "ref": "§1, §6" } }, { "text": { "value": "M2.7 self-evolution: an early operational form where the model autonomously triages failed training runs, edits its own agent scaffold, and runs multi-round self-improvement, closing an expensive human-in-the-loop bottleneck", "provenance": "stated", "ref": "§1, §6.4" }, "ref": { "value": "§1, §6.4", "provenance": "stated", "ref": "§1, §6.4" } }, { "text": { "value": "MTP expansion via weight copying (K=1 → K=3) enabling multi-step speculative decoding, with MTP modules continuously co-trained with the RL policy via top-K KL divergence", "provenance": "stated", "ref": "§2.3, §6.3" }, "ref": { "value": "§2.3, §6.3", "provenance": "stated", "ref": "§2.3, §6.3" } }, { "text": { "value": "A large-scale negative result: full attention retained across all layers after extensive hybrid-SWA exploration (varying ratios, RoPE settings, intra/inter-layer hybrids, sink tokens) showed no sub-quadratic variant reliably matching full attention quality", "provenance": "stated", "ref": "§2.2.2" }, "ref": { "value": "§2.2.2", "provenance": "stated", "ref": "§2.2.2" } } ], "assessment": { "value": "Block-level novelty is modest: fine-grained experts, sigmoid gating with learnable bias (aux-loss-reduced load balancing), and MTP-for-speculative-decoding all follow established DeepSeek-V3/DeepSeekMoE/aux-free lineages the report itself cites. Its distinctive position in the atlas is negative — a deliberate return to full attention at 230B scale while peers (moonshot-kimi-k3, upstage-solar-open-2, zai-glm-5-class hybrids) move to linear-attention hybrids; M2 keeps GQA + RoPE and buys 192K context with staged training rather than sub-quadratic layers. The genuinely novel contributions are training/infra: the Forge RL system (windowed FIFO, prefix-tree merging, white/black-box agent decoupling) and the M2.7 self-evolution loop, neither of which is a block mechanism. MTP weight-copy expansion (K=1→K=3) is a small but real refinement over DeepSeek-V3's single-module MTP.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, moonshot-kimi-k3, upstage-solar-open-2, qwen-qwen3-moe) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "62/62 layers have routed MoE FFNs (256 experts, 8 active per token), §2.1" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "100% of layers are full softmax attention with GQA (no linear/SWA layers in the final model), §2.1, §2.2.2" }, "modality": { "value": "multimodal", "provenance": "derived", "note": "from modality.type (inferred multimodal; MM Claw multi-modal benchmark in evaluation, §7.3)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (public M2.5/M2.7 releases, §7.4); no data release disclosed" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "229.9B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a general frontier-class model 'designed end-to-end for agentic deployment' with broad benchmark coverage across agentic coding, cowork, deep search, office tasks, and reasoning (Abstract, §1). Secondary intent: agentic/long-context (192K), noted in prose." } }, "prose": { "design_overview": { "value": "MiniMax-M2 is a 62-layer, 229.9B-parameter sparse MoE (9.8B active per token) built around the thesis that a mini activation footprint can reach frontier-tier agentic performance. Every block is a full-attention GQA layer (48 query / 8 KV heads, head dim 64, RoPE throughout) followed by an MoE FFN with 256 fine-grained experts, 8 active per token, routed by sigmoid gating with learnable per-expert bias terms that implicitly regulate utilization and let the auxiliary load-balancing loss be greatly reduced. The design is deliberately conservative on the mixing axis: after extensive hybrid sliding-window exploration (spanning ratios, RoPE variants, intra/inter-layer hybrids, and sink tokens), the team kept full attention in all 62 layers, arguing no sub-quadratic variant reliably matched full-attention quality in production settings — a counterpoint to the linear-attention hybrids in the same atlas generation. The 192K native context is reached by staged training (8K → 32K → 192K) rather than any positional-encoding extension. Capacity-per-token economics carry through to inference: a single MTP module (K=1, DeepSeek-V3-style, loss weight 0.3 annealed to 0.1) is expanded by weight copying to three draft modules (K=3) during the decay phase, giving multi-step speculative decoding, and the drafts are co-trained with the RL policy so acceptance survives distribution shift. The bulk of the report is agentic post-training: SFT on interleaved thinking traces, then CISPO agent RL over MDP-formulated trajectories with composite process/outcome/time rewards, run in the Forge system (windowed FIFO, prefix-tree merging up to 40× speedup, prefill-decode disaggregation), culminating in the M2.7 self-evolution checkpoint that debugs its own training runs.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Fine-grained experts — larger number of smaller experts (DeepSeekMoE organization)", "provenance": "stated", "ref": "§2.2.1 (cites dai2024deepseekmoe)" }, "origin": { "value": "DeepSeekMoE", "provenance": "stated", "ref": "§2.2.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) module following the DeepSeek-V3 design during pre-training", "provenance": "stated", "ref": "§2.3 (cites deepseekai2024v3)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "Sigmoid gating with learnable expert bias for auxiliary-loss-free load balancing", "provenance": "stated", "ref": "§2.2.1 (cites wang2024auxfree)" }, "origin": { "value": "Aux-Free MoE (Wang et al., not in atlas)", "provenance": "stated", "ref": "§2.2.1" } }, { "mechanism": { "value": "Clipped Importance Sampling Policy Optimization (CISPO) for RL", "provenance": "stated", "ref": "§6.2.1 (cites minimax2025m1)" }, "origin": { "value": "MiniMax-M1 (not in atlas)", "provenance": "stated", "ref": "§6.2.1" } } ], "influence_out": [], "notable_omissions": { "value": [ "No per-expert FFN dimension disclosed (only 'reducing per-expert FFN size')", "Normalization type and pre/post placement not disclosed; activation function not named", "RoPE base frequency not disclosed", "No optimizer, learning-rate schedule, batch schedule, or training precision disclosed", "No training hardware, FLOPs, or GPU-hours disclosed", "Tokenizer never described (only the 200,064 vocab size)", "Input/output embedding tying not disclosed", "No per-variant configuration for M2.5 / M2.7 — the report treats them as checkpoints of one architecture and states only ~10B activated for M2.7", "Model license not named (the arXiv license on the page covers the paper only)", "The checkpoint-to-checkpoint evolution M2 → M2.5 → M2.7 (data, recipes, reward changes) is not detailed", "No vision/multimodal input pipeline described despite the MM Claw multi-modal evaluation benchmark", "Auxiliary load-balancing loss reduction is not quantified" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "minimax-m2.json" }, { "slug": { "value": "minimax-m3-428b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2606.13392", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/minimax-m3-428b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "MiniMax M3", "provenance": "stated", "ref": "Abstract ('A production-grade natively multimodal model powered by MSA has been publicly released at: https://huggingface.co/MiniMaxAI/MiniMax-M3'); note: the report itself is titled 'MiniMax Sparse Attention' and its experiments use a 109B MSA model, not M3" }, "variants": [ { "value": "10B pilot (ablation model, 16 layers)", "provenance": "stated", "ref": "Appendix B.1" } ], "org": { "value": "MiniMax (with authors from Peking University, NVIDIA, Zhejiang University, HUST, Nanjing University, Hangzhou Dianzi University)", "provenance": "stated", "ref": "Title page author affiliations" }, "release_date": { "value": "2026-06-11", "provenance": "inferred", "note": "arXiv v1 submission date 2026-06-11 per export.arxiv.org API metadata (arXiv:2606.13392); the stored HTML header stamps v2 as 12 Jun 2026" }, "report_url": { "value": "https://arxiv.org/abs/2606.13392", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no model license named in the report (the 'arXiv.org perpetual non-exclusive license' line is the paper's license, not the model's)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('A production-grade natively multimodal model powered by MSA has been publicly released at: https://huggingface.co/MiniMaxAI/MiniMax-M3'); inference kernel open-sourced at github.com/MiniMax-AI/MSA" } }, "scale": { "total_params": { "value": 109, "provenance": "stated", "ref": "§5.1 ('approximately 109B total parameters')", "note": "the report's flagship is the 109B MSA experiment model; the production MiniMax-M3 configuration (428B per the atlas slug) is NOT disclosed anywhere in this report — see prose.notable_omissions" }, "active_params_per_token": { "value": 6, "provenance": "stated", "ref": "§5.1 ('6B activated parameters per token')" }, "layers": { "value": 41, "provenance": "stated", "ref": "§5.1 ('41-layer MoE backbone')" }, "hidden_dim": { "value": 3072, "provenance": "stated", "ref": "§5.1 ('hidden size d_model=3072')" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "expert inner dimension not stated for the 109B model (the 10B pilot states 1536, but that is a different configuration)" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "ffn_inner_dim unknown for the reference variant" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "§5.1 ('64 query heads')" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "§5.1 ('4 KV heads')" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "§5.1 ('head dimension 128')" }, "vocab_size": { "value": 200000, "provenance": "stated", "ref": "§5.1 ('a 200K-token vocabulary'); also Appendix B.1 (pilot)" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "variants": [ { "name": { "value": "10B pilot (ablation)", "provenance": "stated", "ref": "Appendix B.1" }, "total_params": { "value": 10.53, "provenance": "stated", "ref": "Appendix B.1 ('10.53B total parameters')" }, "layers": { "value": 16, "provenance": "stated", "ref": "Appendix B.1" }, "hidden_dim": { "value": 2048, "provenance": "stated", "ref": "Appendix B.1" }, "ffn_inner_dim": { "value": 1536, "provenance": "stated", "ref": "Appendix B.1 ('expert inner dimension 1536')" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Appendix B.1" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "Appendix B.1" }, "context_length": { "value": null, "provenance": "unknown", "note": "not disclosed for the pilot" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "41 layers total, first 3 dense, remaining 38 MoE (38/41 = 92.7% ≥ 80%); the report calls it a MoE backbone" }, "moe": { "expert_count": { "value": 128, "provenance": "stated", "ref": "§5.1 ('128 routed experts')" }, "experts_per_token": { "value": 4, "provenance": "stated", "ref": "§5.1 ('top-4 routed expert selection')" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "§5.1 ('1 shared expert')" }, "routing": { "value": null, "provenance": "unknown", "note": "the expert router function is never described (the report's TopK machinery belongs to the attention Index Branch, not the MoE router)" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "not described" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "not described (no expert inner dimension or latent width given for the 109B model)" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2.3, §3.1 ('MSA, a GQA-based sparse attention mechanism'; 64 query heads, 4 KV heads, head dim 128)", "note": "MSA is exact block-sparse softmax attention built on GQA: a lightweight Index Branch scores key blocks and selects the top-k (k=16, block size 128 → 2,048-token budget per query and GQA group); the Main Branch runs standard scaled dot-product softmax attention over only the selected blocks. The learned sparsity is in the attended support, not in the attention mechanism itself." }, "attention_layer_pattern": { "value": "uniform: every layer uses MSA (all 41 layers; the 3 dense layers and 38 MoE layers share the same MSA attention module)", "provenance": "stated", "ref": "§5.1 ('Each attention module uses MSA')" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no learned attention over prior layer outputs described; standard residual stack" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§5.1 ('RoPE dimension 64')" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE base not stated" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§5.1 ('RoPE dimension 64' with head dimension 128 — RoPE applied to half of the head dims)" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "long-context capability is extended by ~140B tokens of long-context continued training (§5.3) but no positional-encoding modification (RoPE base change, interpolation, etc.) is disclosed" }, "factor": { "value": null, "provenance": "unknown", "note": "no scaling factor disclosed" }, "target_length": { "value": null, "provenance": "unknown", "note": "extension training length not stated; long-context quality is evaluated at 128K (HELMET-128K, RULER-128K) and speedups are measured at 1M context" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "normalization type never named" }, "placement": { "value": null, "provenance": "unknown", "note": "not described" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "MSA scores use plain scaled dot products (index scores divided by sqrt(d_idx), Main Branch by sqrt(d_h)) — standard scaling, not QK-normalisation; the report never describes QK-norm" } }, "activation": { "value": null, "provenance": "unknown", "note": "activation function never named" }, "stability": { "attention_sinks": { "value": false, "provenance": "stated", "ref": "Appendix B.5 ('we do not include the learnable attention sink in the final recipe'); Appendix A (an attention sink on the first token emerges naturally in trained MSA models — emergent, not designed)" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no softcapping described" }, "other": [ { "value": "Forced Local Block: the block containing the query position is always selected during training and inference, reserving one of the k slots and preventing degenerate selections that omit the immediate neighbourhood", "provenance": "stated", "ref": "§3.2 (Local Block)" }, { "value": "Gradient Detach: stop-gradient on the Index Branch input confines the KL loss to the index projections, preventing KL-gradient spikes in the backbone and short-context regression (self-distillation effect)", "provenance": "stated", "ref": "§3.2, Appendix B.3" }, { "value": "Indexer Warmup: a two-stage schedule — full attention with KL-trained index projections for the first iterations (40B tokens at 109B scale), then top-k sparse selection — used for both from-scratch training and full-attention checkpoint conversion", "provenance": "stated", "ref": "§3.2, Appendix B.4" }, { "value": "Emergent attention sink: the learned Index Branch naturally assigns high selection probability to the first block across all heads and layers; a GPT-OSS-style learnable sink was ablated and rejected", "provenance": "stated", "ref": "Appendix A, Appendix B.5" } ] } }, "context": { "trained_length": { "value": null, "provenance": "unknown", "note": "pretraining sequence length never stated; the long-context extension stage runs ~140B tokens and is evaluated at 128K" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract ('reducing per-token attention compute by 28.4× at 1M context'; '14.2× prefill and 7.6× decoding wall-clock speedups at 1M context length')", "note": "1M is the context at which MSA's efficiency claims are made; long-context quality is evaluated at 128K (HELMET/RULER). The production M3 model's context length is not stated in the report." }, "extension_method": { "value": "long-context continued training: ~140B tokens of long-context training on top of the sparse-CPT checkpoint, evaluated on HELMET-128K and RULER-128K with the fixed 2,048-token attention budget preserved", "provenance": "stated", "ref": "§5.3" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer never described (only referenced as shared with the full-scale configuration, Appendix B.1)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 3, "provenance": "stated", "ref": "§5.1 ('total budget of 3T tokens')" }, "data_composition": { "value": "native multimodal mixture of text and image/video data; detailed composition not disclosed", "provenance": "stated", "ref": "§5 ('a native multimodal model trained on a mixture of text and image/video data')" }, "curriculum": { "value": "MSA-PT route: 40B-token indexer-warmup (full attention + KL) then sparse training for the remainder of the 3T budget; MSA-CPT route: starts from a GQA full-attention checkpoint trained on 2.6T tokens, replaces dense attention with MSA, and continues pretraining for 400B tokens (first 40B warmup, then sparse); plus a ~140B-token long-context extension stage", "provenance": "stated", "ref": "§5.1, §5.3" }, "optimizer": { "value": null, "provenance": "unknown", "note": "optimizer never named (Appendix B.1 says the optimizer 'matches the full-scale configuration' but never names it)" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "not described" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "not described" }, "precision": { "value": null, "provenance": "unknown", "note": "not described (kernels support fp8 in addition to bf16 traffic, §4.2, but the training precision is not stated)" }, "parallelism": { "value": null, "provenance": "unknown", "note": "not described" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not stated; H800 appears only in the kernel benchmarks (§4.1, Abstract)" }, "compute": { "value": null, "provenance": "unknown", "note": "not disclosed" } }, "post_training": { "sft": { "value": null, "provenance": "unknown", "note": "the report covers pretraining only; no post-training pipeline is described" }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "§7 (RL post-training is mentioned only as future work: 'extending the same selector-only design to settings beyond pretraining, including reinforcement-learning post-training')" }, "reasoning_training": { "value": "none disclosed", "provenance": "stated", "ref": "§7 (mentioned only as future work)" }, "distillation": { "value": null, "provenance": "unknown", "note": "no distillation described; MSA-CPT is an architecture conversion (dense attention replaced by MSA during continued pretraining), not distillation" } }, "modality": { "type": { "value": "multimodal", "provenance": "stated", "ref": "§5 ('native multimodal model trained on a mixture of text and image/video data'; evaluation includes image (AI2D, ChartQA, MMMU, OCRBench v2, CharXiv, VisualWebBench, CVBench) and video (EgoSchema, LongVideoBench, MLVU, MMVU, VideoMME, TemporalBench) benchmarks)" }, "attachment": { "value": "native", "provenance": "stated", "ref": "Abstract, §5 ('native multimodal training', 'natively multimodal model'); encoder architecture not described in this report" } }, "efficiency": { "kv_cache": { "value": "GQA KV cache proportional to 4 KV heads; MSA sparsifies attention compute (top-16 of 128-token blocks = 2,048-token budget per query) rather than the cache itself — the report does not discuss cache-side savings", "provenance": "derived", "note": "derived from attention_variant (GQA, 4 KV heads) and the MSA selection budget (§3.1, §5.4)" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "§4.2 (kernels 'also support fp8' but no shipped quantization format is described)" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report silent on speculative decoding" }, "serving": { "value": "MSA co-designed GPU kernels (open-sourced at github.com/MiniMax-AI/MSA): exp-free TopK selection (per-thread register top-k with k-element min-heaps and shuffle merge; Bk=128, k=16), KV-outer sparse attention with query gathering and query concatenation for tensor-core utilization, pre-scheduled tile chunking for hot KV blocks, two-phase combine with per-partial LSE buffers (no atomics), LSE fusion into the forward pass, and dynamic load balancing via a persistent grid with atomic work claiming; measured 14.2× prefill and 7.6× decode wall-clock speedups on H800 at 1M context", "provenance": "stated", "ref": "§4, Abstract, §5.4" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 67.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU-Pro (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 38.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BBH (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 66.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA Hard (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 26.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GSM8K (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 77.7, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HumanEval (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 64.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "EvalPlus (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 61.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BigCodeBench (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 44.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "RULER-8K (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 84.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "RULER-32K (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 77.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMMU (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 45.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ChartQA (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 75.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "VideoMME (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 45.48, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MLVU (MSA-PT)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 46.94, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HELMET-128K Overall (MSA-CPT, after long-context extension)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 45.93, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "RULER-128K Overall (MSA-CPT, after long-context extension)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 72.12, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "MSA, a minimal, scalable, and accelerated blockwise sparse attention mechanism built on GQA: a lightweight Index Branch (one index query head per GQA group + one shared index key head, max-pooled block scores) selects a top-k subset of key blocks independently for each GQA group, and the Main Branch runs exact block-sparse softmax attention over the selected blocks", "provenance": "stated", "ref": "Abstract, §3.1" }, "ref": { "value": "Abstract, §3.1", "provenance": "stated", "ref": "Abstract, §3.1" } }, { "text": { "value": "A stable training recipe for the non-differentiable indexer: KL alignment loss against the group-averaged Main Branch distribution, gradient detach confining the auxiliary loss to the index projections, indexer warmup (full-attention first), and a forced local block", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } }, { "text": { "value": "Co-designed GPU kernels — exp-free TopK for the small-k regime, KV-outer sparse attention with pre-scheduled chunking and a two-phase combine (no atomics), LSE fusion, and dynamic load balancing — translating sparsity into 14.2× prefill and 7.6× decode wall-clock speedups on H800 at 1M context", "provenance": "stated", "ref": "§4, Abstract" }, "ref": { "value": "§4, Abstract", "provenance": "stated", "ref": "§4, Abstract" } }, { "text": { "value": "On a 109B-parameter native-multimodal MoE model under a 3T-token budget, MSA matches the GQA full-attention baseline on pretraining and agentic benchmarks while reducing per-token attention compute by 28.4× at 1M context", "provenance": "stated", "ref": "Abstract, §5.3, §7" }, "ref": { "value": "Abstract, §5.3, §7", "provenance": "stated", "ref": "Abstract, §5.3, §7" } }, { "text": { "value": "Two training routes validated: from-scratch sparse pretraining (MSA-PT) and near-lossless conversion of pretrained GQA full-attention checkpoints via sparse continued pretraining (MSA-CPT, 2.6T full-attention + 400B sparse with a 40B warmup)", "provenance": "stated", "ref": "§5.1, §5.2" }, "ref": { "value": "§5.1, §5.2", "provenance": "stated", "ref": "§5.1, §5.2" } }, { "text": { "value": "Open release of the inference kernel (github.com/MiniMax-AI/MSA) and of MiniMax-M3, a production-grade natively multimodal model powered by MSA (huggingface.co/MiniMaxAI/MiniMax-M3)", "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } } ], "assessment": { "value": "MSA is the first learned blockwise-sparse attention record in the atlas: unlike linear-attention hybrids (kimi-k3's KDA, zyphra-zaya1, ai21-jamba) it keeps exact softmax attention and instead learns a per-GQA-group block support, and unlike the atlas' full-attention MoEs (deepseek-v3) it is sub-quadratic. Its closest published relatives — DeepSeek's DSA, NSA, MoBA, InfLLM-V2 — are not in the atlas, so MSA fills a genuinely new cell (learned sparse softmax attention on GQA). The per-group independent Top-k plus block-level selection and the KL+detach+warmup training recipe are the cleanest novel elements; the kernel work is engineering but substantial and open-sourced. Two caveats: all experiments are on the 109B model (the production 428B M3 config is not disclosed in this report, so scale-up behaviour is unverified here), and long-context quality still trails full attention slightly (HELMET-128K −0.60), with the paper itself noting a residual long-context retrieval gap.", "provenance": "inferred", "note": "comparison against atlas entries (moonshot-kimi-k3, zyphra-zaya1-8b, deepseek-v3, ai21-jamba) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "38/41 layers are MoE (first 3 dense), 92.7% ≥ 80%" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "all 41 layers use MSA, a learned blockwise-sparse softmax attention with a fixed 2,048-token budget per query — sub-quadratic in sequence length; classified as efficient-attention (sparse-softmax family) rather than full-attention because the attended support is learned and bounded, though the mechanism is exact softmax over the selected blocks" }, "modality": { "value": "multimodal", "provenance": "derived", "note": "from modality.type (native text + image/video)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (public HF release of MiniMax-M3, open kernel); no training-data release stated" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "109B (reference variant) ≥ 100B threshold; the production M3 (428B per the atlas slug) is not disclosed in this report but would also qualify" }, "design_intent": { "value": "long-context", "provenance": "inferred", "note": "the report frames MSA as enabling ultra-long-context (hundreds of thousands to millions of tokens) at deployment scale ('Ultra-long-context capability is becoming indispensable for frontier LLMs' — Abstract); the model is a generalist multimodal backbone, but the architecture's defining intent is long-context efficiency" } }, "prose": { "design_overview": { "value": "MiniMax Sparse Attention (MSA) is the attention mechanism behind MiniMax M3, described here on the report's 109B/6B-active, 41-layer MoE testbed (3 dense + 38 MoE layers, 128 routed experts with top-4, 1 shared expert, 64 Q heads / 4 KV heads, head dim 128, 200K vocab, native text+image/video training). MSA keeps standard scaled dot-product softmax attention but makes its support learned and bounded: a lightweight Index Branch (one index query head per GQA group plus a single shared index key head) scores key-value blocks by max-pooled dot products and picks the top-k=16 of 128-token blocks for each query and group — the local block is always forced in — giving a fixed 2,048-token budget per query; the Main Branch then runs exact block-sparse attention over only those blocks. Because top-k selection is non-differentiable, the indexer is trained by a KL alignment loss against the group-averaged Main Branch distribution, confined to the index projections by a stop-gradient on its input, and stabilized by a two-stage warmup (full attention first, 40B tokens at this scale) plus the forced local block; ablations showed a learnable attention sink and an index value head are unnecessary, though a first-token sink emerges naturally. Efficiency comes from co-designed kernels: exp-free TopK in the small-k regime and KV-outer sparse attention with pre-scheduled chunking and a two-phase LSE combine for tensor-core utilization, delivering 14.2× prefill / 7.6× decode speedups on H800 at 1M context and a 28.4× attention-FLOPs cut. Two training routes are validated: from-scratch sparse pretraining (MSA-PT) and near-lossless conversion of a GQA full-attention checkpoint via 400B-token sparse CPT (MSA-CPT), plus a ~140B-token long-context extension evaluated at 128K. The production MiniMax-M3 model itself is released on HuggingFace but its configuration is not described in this report.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Grouped-Query Attention backbone — MSA is built on GQA (group-shared index sets, shared KV heads)", "provenance": "stated", "ref": "§2.1 (cites Ainslie et al., 2023); §3.1" }, "origin": { "value": "GQA (Ainslie et al., 2023)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "FlashAttention algorithmic skeleton — the MSA kernels 'reuse the FlashAttention algorithmic skeleton with a loop ordering tuned to the GQA-native, block-granular access pattern'", "provenance": "stated", "ref": "§6 (cites Dao et al., 2022; Dao, 2024)" }, "origin": { "value": "FlashAttention (not in atlas)", "provenance": "stated", "ref": "§6" } } ], "influence_out": [], "notable_omissions": { "value": [ "The production MiniMax-M3 configuration is not disclosed: no 428B figure, no layer count, no context length, no benchmark scores for M3 appear anywhere in the report — it is referenced only by a HuggingFace link in the abstract (the atlas slug's '428b' cannot be sourced from this report)", "All experiments and evaluations are on the 109B MSA testbed, not on the released production model", "Expert inner dimension and MoE router/load-balancing details for the 109B model are not stated (only expert count 128, top-4, 1 shared expert)", "Index head dimension d_idx is used in the equations but its value is never given", "Normalization type, activation function, optimizer, LR schedule, batch size, precision, parallelism, training hardware, and compute are all undisclosed", "Tokenizer algorithm not described (only the 200K vocab size)", "Trained context length of the main pretraining run not stated", "No post-training pipeline described (SFT/RL mentioned only as future work)", "Long-context quality still lags full attention (HELMET-128K Overall −0.60 vs baseline); the paper acknowledges a residual long-context retrieval gap" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "minimax-m3-428b.json" }, { "slug": { "value": "mistral-mixtral-8x7b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2401.04088", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/mistral-mixtral-8x7b", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Mixtral", "provenance": "stated", "ref": "Title / §1" }, "variants": [ { "value": "Mixtral 8x7B", "provenance": "stated", "ref": "§1 Abstract" }, { "value": "Mixtral 8x7B – Instruct", "provenance": "stated", "ref": "§1 Abstract" } ], "org": { "value": "Mistral AI", "provenance": "stated", "ref": "Title page (Code: github.com/mistralai/mistral-src; Webpage: mistral.ai)" }, "release_date": { "value": "2024-01-08", "provenance": "inferred", "note": "arXiv v1 submission date (ID from report URL; arXiv API metadata) — report text itself is undated" }, "report_url": { "value": "https://arxiv.org/abs/2401.04088", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "§1 Abstract / §1" }, "open_weights": { "value": true, "provenance": "stated", "ref": "§1 (\"with open weights\"; \"We release both Mixtral 8x7B and Mixtral 8x7B – Instruct\")" } }, "scale": { "total_params": { "value": 47, "provenance": "stated", "ref": "§1 Abstract (\"each token has access to 47B parameters\")" }, "active_params_per_token": { "value": 13, "provenance": "stated", "ref": "§1 Abstract (\"only uses 13B active parameters during inference\")" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 1 (n_layers)" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 1 (dim)" }, "ffn_inner_dim": { "value": 14336, "provenance": "stated", "ref": "Table 1 (hidden_dim; per-expert FFN inner dimension)" }, "ffn_ratio": { "value": 3.5, "provenance": "derived", "note": "14336 / 4096 = 3.5 (per-expert FFN inner dim ÷ model dim)" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 1 (n_heads)" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1 (n_kv_heads)" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 1 (head_dim)" }, "vocab_size": { "value": 32000, "provenance": "stated", "ref": "Table 1 (vocab_size)" }, "embedding_tied": { "value": null, "provenance": "unknown" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "§1 Abstract / §2.1 (Sparse Mixture of Experts; \"each layer is composed of 8 feedforward blocks (i.e. experts)\")" }, "moe": { "expert_count": { "value": 8, "provenance": "stated", "ref": "Table 1 (num_experts) / §2.1" }, "experts_per_token": { "value": 2, "provenance": "stated", "ref": "Table 1 (top_k_experts) / §2.1 (K=2)" }, "shared_experts": { "value": 0, "provenance": "stated", "ref": "§2.1 (layer output = weighted sum of the two routed experts' outputs; the 8 experts exhaustively compose the layer, no always-on expert described)" }, "routing": { "value": "softmax top-2 over linear-layer logits (G(x) := Softmax(TopK(x·Wg)), K=2)", "provenance": "stated", "ref": "§2.1" }, "load_balancing": { "value": null, "provenance": "unknown" }, "expert_granularity": { "value": "standard FFN experts (vanilla-transformer feedforward blocks, no fine-grained splitting)", "provenance": "stated", "ref": "§2.1 (\"an expert is a standard feedforward block as in a vanilla transformer architecture\")" } }, "attention_variant": { "value": "GQA", "provenance": "inferred", "ref": "Table 1", "note": "Table 1 states n_heads=32 and n_kv_heads=8 (report never names the variant); fewer KV heads than query heads implies grouped-query attention; §2 states attention is fully dense (the sliding-window of Mistral 7B [18] is dropped)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "stated", "ref": "§1 Abstract (\"each layer is composed of 8 feedforward blocks\") / §2 (fully dense context length of 32k)" }, "position_encoding": { "method": { "value": null, "provenance": "unknown" }, "rope_base": { "value": null, "provenance": "unknown" }, "partial_rope": { "value": null, "provenance": "unknown" }, "extension": { "method": { "value": "none", "provenance": "stated", "ref": "§1 / §2 (trained at the fully dense 32k length; no extension disclosed)" }, "factor": { "value": null, "provenance": "n/a", "ref": "§1 / §2", "note": "no context extension; training at the deployed length" }, "target_length": { "value": null, "provenance": "n/a", "ref": "§1 / §2", "note": "no context extension; training at the deployed length" } } }, "normalization": { "type": { "value": null, "provenance": "unknown" }, "placement": { "value": null, "provenance": "unknown" }, "qk_norm": { "value": null, "provenance": "unknown" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§2.1 (\"we use the same SwiGLU architecture as the expert function E_i(x)\")" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown" }, "softcapping": { "value": null, "provenance": "unknown" }, "other": { "value": null, "provenance": "unknown" } }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: report describes its block in detail and never mentions attention over prior layer outputs; standard sequential residual connections are the default mechanism." } }, "context": { "trained_length": { "value": 32768, "provenance": "stated", "ref": "Table 1 (context_len) / §1 Abstract (\"trained with a context size of 32k tokens\")" }, "deployed_length": { "value": 32768, "provenance": "stated", "ref": "§1 (\"context window of 32k tokens\") / Table 1" }, "extension_method": { "value": "none", "provenance": "stated", "ref": "§1 / §2 (trained at the full deployed 32k length; no extension disclosed)" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": null, "provenance": "unknown" }, "data_composition": { "value": "multilingual data; multilingual proportion significantly upsampled relative to Mistral 7B pretraining (no detailed mixture disclosed)", "provenance": "stated", "ref": "§1 / §3.1" }, "curriculum": { "value": "none disclosed", "provenance": "unknown" }, "optimizer": { "value": null, "provenance": "unknown" }, "lr_schedule": { "value": null, "provenance": "unknown" }, "batch_schedule": { "value": null, "provenance": "unknown" }, "precision": { "value": null, "provenance": "unknown" }, "parallelism": { "value": null, "provenance": "unknown" }, "hardware": { "value": null, "provenance": "unknown" }, "compute": { "value": null, "provenance": "unknown" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4 (\"supervised fine-tuning (SFT) on an instruction dataset\")", "note": "applies to the Mixtral 8x7B – Instruct variant" }, "preference_optimization": { "value": "DPO", "provenance": "stated", "ref": "§4 (\"Direct Preference Optimization (DPO) [25] on a paired feedback dataset\")" }, "reasoning_training": { "value": null, "provenance": "unknown" }, "distillation": { "value": null, "provenance": "unknown" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "decoder-only LM; all described inputs/outputs and evaluations are text (MMLU, code, math, multilingual QA, bias benchmarks); no non-text modality described" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§1 (decoder-only text language model)" } }, "efficiency": { "kv_cache": { "value": "GQA: KV cache scales with 8 KV heads rather than 32 query heads", "provenance": "derived", "note": "derived from attention_variant GQA (n_kv_heads=8, n_heads=32, Table 1)" }, "quantization": { "value": "none disclosed", "provenance": "unknown" }, "speculative_dedup": { "value": "none disclosed", "provenance": "unknown" }, "serving": { "value": "vLLM integration with Megablocks CUDA kernels for efficient MoE inference; TensorRT-LLM/Triton support for sparse MoE (acknowledgements); Skypilot deployment of vLLM endpoints", "provenance": "stated", "ref": "§1 / Acknowledgements" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "70.6%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HellaSwag", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "84.4%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Winogrande", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "77.2%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "PIQA", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "83.6%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ARC-Easy", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "83.1%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ARC-Challenge", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "59.7%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "NaturalQuestions", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "30.6%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "TriviaQA", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "71.5%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HumanEval", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "40.2%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MBPP", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "60.7%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MATH", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "28.4%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GSM8K", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "74.4%", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU (MCQ in 57 subjects)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "70.6%", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HellaSwag (10-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "86.7%", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ARC Challenge (25-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "85.8%", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "WinoGrande (5-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "81.2%", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MBPP (pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "60.7%", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GSM-8K (5-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "58.4%", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MT Bench (for Instruct Models)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "8.30", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ARC-Challenge (French)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "58.2%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HellaSwag (French)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "77.4%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU (French)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "70.9%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "ARC-Challenge (German)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "54.3%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HellaSwag (German)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "73.0%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU (German)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "71.5%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "ARC-Challenge (Spanish)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "55.4%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HellaSwag (Spanish)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "77.6%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU (Spanish)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "72.5%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "ARC-Challenge (Italian)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "52.8%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HellaSwag (Italian)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "75.1%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU (Italian)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "70.9%", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "BBQ accuracy", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "56.0%", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BOLD sentiment score (gender)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "0.323 ± 0.045", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BOLD sentiment score (profession)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "0.243 ± 0.087", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BOLD sentiment score (religious_ideology)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "0.144 ± 0.089", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BOLD sentiment score (political_ideology)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "0.186 ± 0.146", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BOLD sentiment score (race)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "0.232 ± 0.052", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Passkey retrieval", "provenance": "stated", "ref": "§3.2 / Figure 4 (Left)" }, "value": { "value": "100%", "provenance": "stated", "ref": "§3.2 / Figure 4 (Left)" }, "ref": { "value": "§3.2 / Figure 4 (Left)", "provenance": "stated", "ref": "§3.2 / Figure 4 (Left)" } }, { "name": { "value": "LMSys Arena Elo (Instruct v0.1)", "provenance": "stated", "ref": "Figure 6" }, "value": { "value": "1121", "provenance": "stated", "ref": "Figure 6" }, "ref": { "value": "Figure 6", "provenance": "stated", "ref": "Figure 6" } } ] }, "contributions": { "claimed": [ { "text": { "value": "First sparse mixture-of-experts network to reach state-of-the-art performance among open-source models.", "provenance": "stated", "ref": "§6 Conclusion / §1" }, "ref": { "value": "§6 Conclusion / §1", "provenance": "stated", "ref": "§6 Conclusion / §1" } }, { "text": { "value": "Matches or outperforms Llama 2 70B and GPT-3.5 on most benchmarks while activating only 13B parameters per token (5x fewer active parameters than Llama 2 70B).", "provenance": "stated", "ref": "§1 Abstract / Table 2" }, "ref": { "value": "§1 Abstract / Table 2", "provenance": "stated", "ref": "§1 Abstract / Table 2" } }, { "text": { "value": "Vastly outperforms Llama 2 70B on mathematics, code generation, and multilingual benchmarks.", "provenance": "stated", "ref": "§1 / §3 / Table 4" }, "ref": { "value": "§1 / §3 / Table 4", "provenance": "stated", "ref": "§1 / §3 / Table 4" } }, { "text": { "value": "Mixtral 8x7B – Instruct (SFT + DPO) surpasses GPT-3.5 Turbo, Claude-2.1, Gemini Pro, and Llama 2 70B chat on human evaluation; MT-Bench 8.30, best open-weights model as of December 2023.", "provenance": "stated", "ref": "§4 / Table 3" }, "ref": { "value": "§4 / Table 3", "provenance": "stated", "ref": "§4 / Table 3" } }, { "text": { "value": "Fully dense 32k-token context with 100% passkey retrieval regardless of sequence length and passkey position.", "provenance": "stated", "ref": "§1 / §3.2 / Figure 4" }, "ref": { "value": "§1 / §3.2 / Figure 4", "provenance": "stated", "ref": "§1 / §3.2 / Figure 4" } }, { "text": { "value": "Open release under Apache 2.0 with open-source inference-stack contributions (vLLM + Megablocks kernels, TensorRT-LLM support).", "provenance": "stated", "ref": "§1 / Acknowledgements" }, "ref": { "value": "§1 / Acknowledgements", "provenance": "stated", "ref": "§1 / Acknowledgements" } } ], "assessment": { "value": "The architectural novelty is modest: the report itself frames Mixtral as Mistral 7B's architecture with every FFN replaced by a GShard-style top-2 routed MoE layer (both GShard [21] and Mistral 7B [18] are named as sources), with sliding-window attention dropped for fully dense 32k attention. Its contribution is primarily demonstrative and empirical — the first open-weights MoE to match a dense 70B-class model with roughly 5x fewer active parameters, plus an unusually candid routing analysis showing experts do not specialize by domain and that expert choice exhibits syntactic structure and temporal locality. Relative to other atlas entries it introduces no new attention or positional-encoding mechanism; its influence rests on popularising sparse-MoE cost-performance at the 7B-expert scale rather than on a novel mechanism.", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all 32 FFN sub-blocks are routed MoE layers (§2.1) → 100% of layers, ≥80% rule" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "fully dense attention over the 32k context at every layer (§2); no sliding-window or sparse pattern" }, "modality": { "value": "text-only", "provenance": "derived", "note": "derived from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "weights released under Apache 2.0 (§1); training data not released (only multilingual composition mentioned)" }, "scale_class": { "value": "large", "provenance": "derived", "note": "total_params 47B → 10B ≤ p < 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a general-purpose open-weight model matching/beating dense 70B and GPT-3.5 across broad benchmark coverage (abstract, §1, §3)" } }, "prose": { "design_overview": { "value": "Mixtral 8x7B is a decoder-only sparse mixture-of-experts transformer in which every one of the 32 layers replaces its feedforward sub-block with an MoE layer holding 8 expert FFNs. For each token, a linear router produces logits x·Wg, keeps the top 2, and softmaxes them; the layer output is the weighted sum of the two selected experts' outputs, so the selected experts can differ per token and per layer. Each expert is a standard vanilla-transformer FFN using the SwiGLU activation, with inner dimension 14336. The rest of the block follows the Mistral 7B design the report explicitly inherits — grouped-query attention (32 query heads, 8 KV heads, head dimension 128) — with the stated exceptions that attention is fully dense over the 32768-token context (no sliding window) and the FFNs are routed. Total parameters are 47B, of which only about 13B are active per token, roughly 5x fewer than Llama 2 70B, which the model matches or beats across the report's benchmarks, especially in math, code, and multilingual tasks. The report is candid that the MoE formulation is close to GShard, differing by replacing every FFN block (GShard alternates) and using simpler gating for the second expert. Serving efficiency relies on Megablocks-style sparse matmul kernels integrated into vLLM, and expert parallelism for multi-GPU placement. A routing analysis finds experts do not specialize by domain; assignments track syntax and show temporal locality, relevant to expert-parallel load balancing.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Sparse MoE layer with softmax top-k gating replacing FFN sub-blocks (all layers, vs GShard's alternating pattern)", "provenance": "stated", "ref": "§2.1" }, "origin": { "value": "GShard [21]", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Base decoder-only transformer architecture and its modifications (report: \"same architecture as Mistral 7B\", \"same modifications as described in [18]\")", "provenance": "stated", "ref": "§1 Abstract / §2" }, "origin": { "value": "Mistral 7B [18]", "provenance": "stated", "ref": "§1 Abstract / §2" } }, { "mechanism": { "value": "Softmax over Top-K logits of a linear layer as the expert gating function", "provenance": "stated", "ref": "§2.1" }, "origin": { "value": "Shazeer et al., sparsely-gated MoE [28]", "provenance": "stated", "ref": "§2.1" } } ], "influence_out": [], "notable_omissions": { "value": [ "No pretraining token count or detailed data mixture (only \"multilingual data\" with multilingual proportion upsampled relative to Mistral 7B).", "No training hyperparameters: optimizer, LR schedule, batch size, precision, hardware, or compute (FLOPs/GPU-hours) are not disclosed.", "No tokenizer details beyond vocab size 32000 (algorithm not named).", "No router load-balancing loss or mechanism disclosed (load balancing discussed only as an expert-parallelism serving concern).", "Core architectural modifications (positional encoding, normalization, placement) deferred to Mistral 7B [18] without being named in this report.", "Attention variant (GQA) not named; inferable only from the n_heads/n_kv_heads counts in Table 1.", "Instruct fine-tuning details (SFT and DPO dataset composition/size) not disclosed.", "No quantization formats or speculative-decoding support disclosed." ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "mistral-mixtral-8x7b.json" }, { "slug": { "value": "moonshot-kimi-k2", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2507.20534", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/moonshot-kimi-k2/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Kimi K2", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "Kimi Team (Moonshot AI)", "provenance": "stated", "ref": "Title byline; fn 1 (github.com/MoonshotAI/checkpoint-engine); Appendix C ('Moonshot's in-house agentic framework')" }, "release_date": { "value": "2025-07-28", "provenance": "inferred", "note": "Report text is undated (LMSYS Arena 'July 17, 2025' is a benchmark snapshot, not a publication date); arXiv v1 submission date 2025-07-28 per export.arxiv.org API metadata (arXiv:2507.20534)" }, "report_url": { "value": "https://arxiv.org/abs/2507.20534", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We release our base and post-trained model checkpoints'; fn 1 (huggingface.co/moonshotai/Kimi-K2-Instruct)" } }, "scale": { "total_params": { "value": 1043, "provenance": "stated", "ref": "Table 4 (# Total Params 1043B)", "note": "Abstract rounds to 1T; §1 and Table 2 print 1.04T" }, "active_params_per_token": { "value": 32.6, "provenance": "stated", "ref": "Table 2 (Activated Parameters 32.6B)", "note": "Abstract and §1 round to 32B" }, "layers": { "value": 61, "provenance": "stated", "ref": "Table 2 (#Layers 61)" }, "hidden_dim": { "value": 7168, "provenance": "stated", "ref": "§2.3 (model hidden dimension of 7168)" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "ref": "§2.3 (MoE expert hidden dimension of 2048)", "note": "per-expert inner dimension" }, "ffn_ratio": { "value": 0.29, "provenance": "derived", "note": "2048 / 7168 = 0.2857 (per-expert inner dim over hidden dim)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "§2.3, Table 2 (Attention Heads 64)" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§2.3 (MLA: keys/values not materialized per head; low-rank latent KV compression)" }, "head_dim": { "value": 112, "provenance": "derived", "note": "7168 / 64 = 112 (query head dim; MLA latent KV and rotary dims are not disclosed)" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "vocabulary size never stated in the report" }, "embedding_tied": { "value": null, "provenance": "unknown" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "61 layers total, 1 dense layer (Table 2) → 60/61 = 98.4% of layers have routed MoE FFNs" }, "moe": { "expert_count": { "value": 384, "provenance": "stated", "ref": "§2.3, Table 2 (Experts total 384)" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§2.3, Table 2 (Experts Active per Token 8)" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "Table 2 (Shared Experts 1)" }, "routing": { "value": null, "provenance": "unknown", "note": "router gating function (sigmoid/softmax) not described; only counts and sparsity are given" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no load-balancing mechanism described (unlike DeepSeek-V3's aux-loss-free bias)" }, "expert_granularity": { "value": "ungrouped experts at 2048 width — Table 2 lists 'Expert Grouping: No' (vs DeepSeek-V3's 'Yes'); 384 routed experts, no grouping", "provenance": "inferred", "note": "evidence: Table 2 (Expert Grouping row); report never uses the word 'fine-grained'" } }, "attention_variant": { "value": "MLA", "provenance": "stated", "ref": "§2.3 ('employing Multi-head Latent Attention (MLA) as the attention mechanism')" }, "attention_layer_pattern": { "value": "uniform — MLA in every layer; 1 of the 61 layers has a dense (non-routed) FFN per Table 2, otherwise no attention-type alternation", "provenance": "stated", "ref": "§2.3, Table 2 (Number of Dense Layers 1)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no learned attention over prior layer outputs described (§2.3)" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1 (MLA decomposed into compressed qC/kC and rotary qR/kR components); §2.5 (YaRN extension applied to the rotary encoding)" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE base frequency not disclosed" }, "partial_rope": { "value": true, "provenance": "inferred", "note": "evidence: §2.1 MLA decomposition into compressed (qC/kC) and rotary (qR/kR) components implies RoPE applied to a subset of head dims, DeepSeek-V2-style MLA" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "ref": "§2.5 ('To extend the context window to 128k, we employed the YaRN method')" }, "factor": { "value": null, "provenance": "unknown", "note": "YaRN scaling factor not disclosed" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§2.5 (context window extended to 128k)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "Appendix E ('RMS-Norm keeps ‖xi‖‖xj‖ bounded'); §2.4.3 names 'LayerNorm' for recomputation, presumably the same norm" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": false, "provenance": "stated", "ref": "§2.1 (QK-Norm 'is not applicable to MLA'; Kimi K2 instead applies QK-Clip weight clipping)" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§2.4.3 (recomputation of 'SwiGLU'; FP8 storage for 'inputs of MoE up-projections and SwiGLU')" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described (§2.3)" }, "softcapping": { "value": false, "provenance": "stated", "ref": "§2.1 (logit soft-cap described as an insufficient mitigation — 'the dot products between queries and keys can still grow excessively before capping is applied'; K2 uses QK-Clip instead)" }, "other": [ { "value": "QK-Clip: per-head rescaling of query/key projection weights (threshold τ = 100) post-update, applied only to unshared MLA components (qC/kC scaled by √γ, qR by γ, shared kR untouched); transiently active — 12.7% of heads triggered it in the first 70,000 steps, then self-deactivated", "provenance": "stated", "ref": "§2.1, Appendix D" }, { "value": "FP8-E4M3 storage (1×128 tiles with FP32 scales) for inputs of MoE up-projections and SwiGLU to cut activation memory; FP8 deliberately not used in computation", "provenance": "stated", "ref": "§2.4.3" }, { "value": "Selective recomputation (LayerNorm, SwiGLU, MLA up-projections, MoE down-projections) plus CPU offload of remaining activations with a copy engine overlapping offload/onload", "provenance": "stated", "ref": "§2.4.3" } ] } }, "context": { "trained_length": { "value": 4096, "provenance": "stated", "ref": "§2.5 ('We pre-trained the model with a 4,096-token context window')" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§2.5 ('To extend the context window to 128k, we employed the YaRN method')" }, "extension_method": { "value": "YaRN after an annealing phase (400B tokens at 4k, then 60B tokens at 32k sequence) and a long-context activation stage", "provenance": "stated", "ref": "§2.5" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer algorithm and vocabulary size never described" }, "notes": { "value": "No tokenizer details disclosed; Appendix B documents the tool-calling token template (<|im_begin|>, <|tool_call_section_begin|>, <|tool_call_begin|>, <|tool_arguments_begin|>, <|tool_call_end|>, <|im_end|>) with TypeScript tool declarations and a constrained-decoding 'enforcer' module", "provenance": "stated", "ref": "Appendix B" } }, "training": { "tokens": { "value": 15.5, "provenance": "stated", "ref": "Abstract, §2 ('pre-trained on 15.5 trillion tokens')" }, "data_composition": { "value": "15.5T tokens of curated high-quality data over four domains — Web Text, Code, Mathematics, Knowledge; most pipelines follow Kimi K1.5; Knowledge rephrased with WRAP-inspired style-/perspective-diverse prompting, chunk-wise autoregressive rewriting and fidelity verification; Mathematics rewritten into SwallowMath-style learning notes and translated into English; each corpus rephrased at most twice", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "annealing phase (400B tokens at 4k sequence, then 60B tokens at 32k) followed by a long-context activation stage; YaRN extension to 128k; batch size held constant throughout", "provenance": "stated", "ref": "§2.5" }, "optimizer": { "value": "MuonClip: Muon with weight decay, consistent update RMS scaling, and per-head QK-Clip (τ = 100)", "provenance": "stated", "ref": "§2.1" }, "lr_schedule": { "value": "WSD: 500-step warm-up then constant 2e-4 for the first 10T tokens, cosine decay 2e-4 → 2e-5 over the remaining 5.5T; annealing decays 2e-5 → 7e-6", "provenance": "stated", "ref": "§2.5" }, "batch_schedule": { "value": "global batch size held at 67M tokens throughout pre-training and annealing", "provenance": "stated", "ref": "§2.5" }, "precision": { "value": "parameters stored in BF16 with FP32 gradient-accumulation buffer; FP8-E4M3 used only for storing insensitive activations (1×128 tiles, FP32 scales), never in computation", "provenance": "stated", "ref": "§2.4.2, §2.4.3" }, "parallelism": { "value": "16-way Pipeline Parallelism with virtual stages (interleaved 1F1B; weight-gradient computation decoupled from backward and overlapped with PP communication), 16-way Expert Parallelism, ZeRO-1 Data Parallelism; EP all-to-all overlapped with computation; trainable on any node count multiple of 32", "provenance": "stated", "ref": "§2.4.2" }, "hardware": { "value": "NVIDIA H800 cluster: 8 GPUs per node with NVLink/NVSwitch and 2 TB RAM per node, 8×400 Gbps RoCE inter-node interconnect", "provenance": "stated", "ref": "§2.4.1" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (large-scale instruction-tuning dataset; agentic tool-use data synthesis pipeline with 3,000+ real MCP tools and 20,000+ synthetic tools)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§3.2.3 (Kimi K1.5 policy-optimization objective with mean-reward baseline and KL regularization; RLVR for verifiable tasks plus Self-Critique Rubric Reward for subjective tasks; no RLHF/PPO/DPO/GRPO named)" }, "reasoning_training": { "value": "joint RL stage combining verifiable rewards (math/STEM/logic, complex instruction following, faithfulness via a sentence-level judge, coding/SWE, safety) with self-critique rubric reward; per-sample token budget control, PTX loss, temperature decay; long-horizon agentic rollout in real and synthetic environments with partial rollouts", "provenance": "stated", "ref": "§3.2, §3.3.4" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "SFT candidate responses were generated with Kimi K1.5 and in-house domain-specialized expert models then filtered by LLM/human judges (§3.1) — teacher-generated training data, not distillation per schema rule; noted in prose" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: no multimodal components described anywhere; all evaluation benchmarks are text-only; report frames K2 as a non-thinking text LLM" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": "MLA low-rank latent KV cache (keys/values reconstructed from a compressed per-token latent, DeepSeek-V3-style); attention heads deliberately cut from 128 to 64 to reduce long-context inference overhead (83% inference-FLOP increase estimated for 128 heads at 128k)", "provenance": "derived", "note": "consequence of attention_variant = MLA plus §2.3 head-count rationale" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "FP8-E4M3 appears only as training-time activation storage (§2.4.3), not shipped inference quantization" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding or MTP described" }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "no serving stack described; inference-motivated design choices (head count, sparsity 48) are architectural, not a serving system" } }, "evaluation": { "benchmarks": [ { "name": { "value": "Tau2-Bench (Pass@1)", "provenance": "stated", "ref": "Abstract" }, "value": { "value": 66.1, "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "name": { "value": "ACEBench (En) (Acc.)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 76.5, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "SWE-bench Verified — Agentic Single Attempt (Pass@1)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 65.8, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "SWE-bench Multilingual (Pass@1)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 47.3, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "LiveCodeBench v6 (Pass@1)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 53.7, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "AIME 2025 (Avg@64)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 49.5, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "GPQA-Diamond (Avg@8)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 75.1, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "OJBench (Pass@1)", "provenance": "stated", "ref": "Abstract, Table 3" }, "value": { "value": 27.1, "provenance": "stated", "ref": "Abstract, Table 3" }, "ref": { "value": "Abstract, Table 3", "provenance": "stated", "ref": "Abstract, Table 3" } }, { "name": { "value": "SWE-bench Verified — Agentic Multi-Attempt (Pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 71.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-bench Verified — Agentless Single Patch (Pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 51.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MultiPL-E (Pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 85.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Multi-SWE-bench (Pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 18.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Lancer (Pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 39.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "PaperBench Code-Dev (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 27.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Terminal Bench Terminus (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 25.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Terminal Bench In-House (Acc.)", "provenance": "stated", "ref": "Table 3, Appendix C" }, "value": { "value": 30.0, "provenance": "stated", "ref": "Table 3, Appendix C" }, "ref": { "value": "Table 3, Appendix C", "provenance": "stated", "ref": "Table 3, Appendix C" } }, { "name": { "value": "Aider-Polyglot (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 60.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Tau2 retail (Avg@4)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 70.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME 2024 (Avg@64)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 69.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MATH-500 (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 97.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HMMT 2025 (Avg@32)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 38.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ZebraLogic (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AutoLogi (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SuperGPQA (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 57.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU (EM)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU-Redux (EM)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 92.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU-Pro (EM)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 81.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "IFEval (Prompt Strict)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Multi-Challenge (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 54.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SimpleQA (Correct)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 31.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LiveBench (Pass@1)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 76.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Arena Hard v2.0 Hard Prompt (Win rate)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 54.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Arena Hard v2.0 Creative Writing (Win rate)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 85.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "FACTS Grounding (Adjusted)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 88.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HHEM v2.1 (1-Hallu.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 98.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "FaithJudge (1-Hallu.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 92.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "DROP (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 93.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MRCR (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 55.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "FRAMES (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 77.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LongBench v2 (Acc.)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 49.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU (Base, 5-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 87.79, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "EvalPlus (Base)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 80.33, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } } ] }, "contributions": { "claimed": [ { "text": { "value": "MuonClip: a novel optimizer integrating the token-efficient Muon algorithm with a QK-Clip stability mechanism (per-head rescaling of query/key projection weights), enabling pre-training on 15.5T tokens with zero loss spikes", "provenance": "stated", "ref": "Abstract, §2.1" }, "ref": { "value": "Abstract, §2.1", "provenance": "stated", "ref": "Abstract, §2.1" } }, { "text": { "value": "A large-scale agentic data synthesis pipeline that systematically generates tool-use demonstrations via simulated and real-world environments (3,000+ real MCP tools, 20,000+ synthetic tools, rubric-based task generation, LLM-judge filtering)", "provenance": "stated", "ref": "Abstract, §3.1.1" }, "ref": { "value": "Abstract, §3.1.1", "provenance": "stated", "ref": "Abstract, §3.1.1" } }, { "text": { "value": "A joint reinforcement learning framework combining verifiable rewards (RLVR) with a Self-Critique Rubric Reward mechanism that extends alignment from static tasks into open-ended domains", "provenance": "stated", "ref": "Abstract, §3.2.2" }, "ref": { "value": "Abstract, §3.2.2", "provenance": "stated", "ref": "Abstract, §3.2.2" } }, { "text": { "value": "A sparsity scaling law for the MoE family under Muon showing that sparsity 48 cuts FLOPs by 1.69×/1.39×/1.15× vs sparsity 8/16/32 at equal loss, motivating 384 experts with 8 active", "provenance": "stated", "ref": "§2.3" }, "ref": { "value": "§2.3", "provenance": "stated", "ref": "§2.3" } }, { "text": { "value": "State-of-the-art performance among open-source non-thinking models, with strengths in agentic capabilities (Tau2-Bench 66.1, ACEBench 76.5, SWE-bench Verified 65.8) and top-1 open-source ranking on LMSYS Arena (July 17, 2025)", "provenance": "stated", "ref": "Abstract, §1, §4.1.2" }, "ref": { "value": "Abstract, §1, §4.1.2", "provenance": "stated", "ref": "Abstract, §1, §4.1.2" } }, { "text": { "value": "Open release of base and post-trained checkpoints plus the checkpoint-engine source code to facilitate agentic-intelligence research", "provenance": "stated", "ref": "Abstract, §3.3.2" }, "ref": { "value": "Abstract, §3.3.2", "provenance": "stated", "ref": "Abstract, §3.3.2" } } ], "assessment": { "value": "Architecturally, K2 is a direct refinement of DeepSeek-V3 (the closest atlas peer, deepseek-v3): the same MLA attention and shared-expert MoE skeleton, with sparsity pushed to 48 (384 routed + 1 shared expert vs 256 + 1), attention heads halved 128→64, expert grouping removed, and one unexplained dense layer among 61. The genuinely novel contributions sit at the optimizer and data/RL level: QK-Clip is a distinct stability mechanism (post-update per-head weight rescaling) that the report explicitly contrasts with both logit soft-capping and QK-Norm, and the agentic data-synthesis + self-critique rubric RL pipeline is a training contribution, not a block change. The 15.5T-token zero-loss-spike claim is internally validated only by a loss curve. Within the atlas, K2 is the direct ancestor of moonshot-kimi-k3, whose report inherits the weight-clipping lineage and rephrasing pipelines.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, moonshot-kimi-k3, deepseek-r1) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "60/61 layers routed MoE (1 dense layer, Table 2) = 98.4% ≥ 80%" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "MLA (full softmax attention with latent KV compression) in every layer; no linear-attention or SSM layers anywhere" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data composition described (four domains + rephrasing) but corpus not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "1043B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as 'open agentic intelligence' — a general frontier model (coding, math, STEM, knowledge, agentic tool use, Chinese) evaluated in non-thinking mode; agentic emphasis is the differentiator rather than a narrow specialization" } }, "prose": { "design_overview": { "value": "Kimi K2 is a 61-layer, 1.04T-parameter ultra-sparse MoE (32.6B active per token) that largely follows DeepSeek-V3's blueprint: Multi-head Latent Attention in every layer (latent KV compression, partial rotary encoding), a hidden width of 7168, and a shared-expert MoE — but with sparsity scaled up to 48 via a dedicated scaling law (384 routed experts of width 2048, 8 active per token, 1 shared expert), attention heads halved to 64 to cut long-context inference FLOPs, and expert grouping dropped. The headline novelty is MuonClip: the Muon optimizer (inherited from Moonlight) augmented with per-head QK-Clip, which rescales query/key projection weights whenever max attention logits exceed τ=100, stabilizing Muon's tendency to explode logits and yielding a spike-free 15.5T-token run (4k context, WSD schedule, 67M-token batches, annealing at 4k/32k, then YaRN to 128k). Post-training is a three-stage story: SFT on a large instruction set with a large-scale agentic tool-use synthesis pipeline (real MCP tools plus evolved synthetic tools, rubric-verified trajectories), then joint RL combining verifiable rewards with a self-critique rubric reward and per-sample budget control, PTX loss, and temperature decay, with long-horizon agentic rollout in real sandboxes. The report positions K2 as the strongest open non-thinking model, particularly on SWE and tool-use. In the atlas, K2 is the direct ancestor of moonshot-kimi-k3, whose report explicitly carries over K2's weight-clipping and data pipelines; note this record's own report is text-only and predates the K3 report by a year.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Multi-head Latent Attention (MLA) with partial rotary encoding", "provenance": "stated", "ref": "§2.3 (architecture 'similar design to DeepSeek-V3 [11]', MLA [45] = DeepSeek-V2)" }, "origin": { "value": "DeepSeek-V2 / DeepSeek-V3", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "Ultra-sparse MoE with one shared expert and 8 routed experts active", "provenance": "stated", "ref": "§2.3, Table 2 (direct comparison against DeepSeek-V3's 256/8/1)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "Muon optimizer with consistent update RMS scaling", "provenance": "stated", "ref": "§2.1 (cites [47] 'Muon is scalable for LLM training' — Moonlight)" }, "origin": { "value": "Moonlight (Moonshot)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "WSD (warmup-stable-decay) learning-rate schedule", "provenance": "stated", "ref": "§2.5 (cites MiniCPM [26])" }, "origin": { "value": "MiniCPM", "provenance": "stated", "ref": "§2.5" } }, { "mechanism": { "value": "YaRN context extension to 128k", "provenance": "stated", "ref": "§2.5 (cites [56])" }, "origin": { "value": "YaRN (Peng et al.)", "provenance": "stated", "ref": "§2.5" } }, { "mechanism": { "value": "K1.5 policy-optimization RL objective with mean-reward baseline and KL regularization", "provenance": "stated", "ref": "§3.2.3 (cites Kimi K1.5 [36])" }, "origin": { "value": "Kimi K1.5", "provenance": "stated", "ref": "§3.2.3" } }, { "mechanism": { "value": "Style-/perspective-diverse knowledge rephrasing (WRAP-inspired) and learning-note math rewriting (SwallowMath-style)", "provenance": "stated", "ref": "§2.2 (cites WRAP [50], SwallowMath [16])" }, "origin": { "value": "WRAP / SwallowMath", "provenance": "stated", "ref": "§2.2" } } ], "influence_out": [ { "model_slug": { "value": "moonshot-kimi-linear-48b-a3b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Pretraining recipe from Kimi K2: MuonClip optimizer, WSD schedule, annealing + long-context activation phase, K2 pretraining corpus", "provenance": "inferred", "note": "mechanism as stated in moonshot-kimi-linear-48b-a3b report's lineage_in" } } ], "notable_omissions": { "value": [ "Vocabulary size and tokenizer algorithm not disclosed (only the tool-calling token template appears, Appendix B)", "Router gating function and load-balancing mechanism not described (only expert counts and sparsity 48)", "MLA latent dimensions (compressed-KV and rotary widths) not disclosed; head dim only derivable as 7168/64 = 112", "RoPE base frequency and YaRN scaling factor not disclosed", "Training compute (FLOPs or GPU-hours) not disclosed", "No license named for the released checkpoints", "Input/output embedding tying not disclosed", "Normalization placement (pre/post) not described", "The single dense layer among 61 layers is never explained", "Per-domain proportions of the 15.5T-token corpus not given", "No ablation of the QK-Clip threshold τ on the full model (τ=100 only)", "Inference serving stack (batching, quantization, speculative decoding) not described; FP8 appears only as training-time activation storage" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "moonshot-kimi-k2.json" }, { "slug": { "value": "moonshot-kimi-k3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2607.24653", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-08", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/moonshot-kimi-k3/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Kimi K3", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "Kimi Team (Moonshot AI)", "provenance": "stated", "ref": "Title page byline; fn 1 (huggingface.co/moonshotai/Kimi-K3)" }, "release_date": { "value": "2026-07-27", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date 2026-07-27 per export.arxiv.org API metadata (arXiv:2607.24653)" }, "report_url": { "value": "https://arxiv.org/abs/2607.24653", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We release the full Kimi K3 model weights'; fn 1 (HuggingFace)" } }, "scale": { "total_params": { "value": 2780, "provenance": "stated", "ref": "Table 1", "note": "Table 1 prints 2.78T; abstract rounds to 2.8T" }, "active_params_per_token": { "value": 104.2, "provenance": "stated", "ref": "Table 1", "note": "Table 1 prints 104.2B; abstract rounds to 104B" }, "layers": { "value": 93, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": 7168, "provenance": "stated", "ref": "Table 1" }, "ffn_inner_dim": { "value": 3072, "provenance": "stated", "ref": "Table 1 (MoE Hidden Dimension per Expert)" }, "ffn_ratio": { "value": 0.4, "provenance": "derived", "note": "3072 / 7168 = 0.43 (per-expert inner dim over hidden dim; routed experts operate in latent width 3584, not full width)" }, "attention_heads_q": { "value": 96, "provenance": "stated", "ref": "Table 1 (Attention Heads)" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§2.1.2", "note": "MLA layers cache a low-rank latent (no KV heads); KDA layers keep a recurrent state (no KV heads)" }, "head_dim": { "value": null, "provenance": "unknown", "note": "hidden 7168 / 96 heads is non-integer (74.67), so the schema derivation is invalid; report does not state head dims or KDA state dims (d_k, d_v)" }, "vocab_size": { "value": 160000, "provenance": "stated", "ref": "Table 1 (Vocabulary Size 160K)" }, "embedding_tied": { "value": null, "provenance": "unknown" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "93 layers total, 1 dense layer (Table 1) → 92/93 = 98.9% of layers have routed MoE FFNs (each attention layer is paired with a Stable LatentMoE, §2)" }, "moe": { "expert_count": { "value": 896, "provenance": "stated", "ref": "Table 1 (Routed Experts)" }, "experts_per_token": { "value": 16, "provenance": "stated", "ref": "Table 1 (Experts Active per Token)" }, "shared_experts": { "value": 2, "provenance": "stated", "ref": "§2.3 (N_s = 2 full-width shared experts in every layer)" }, "routing": { "value": "sigmoid router scores with biased top-k selection; mixture weights renormalized over selected experts (bias excluded from weights)", "provenance": "stated", "ref": "§2.3.3 Eq. 13 (s_i = Sigmoid(W_r x_i); argtop_k(s + b))" }, "load_balancing": { "value": "Quantile Balancing (QB): auxiliary-loss-free per-expert bias set from the (1 − k/n)-quantile of router-score margins, estimated from per-expert histograms over the global batch; bias frozen at inference", "provenance": "stated", "ref": "§2.3.3" }, "expert_granularity": { "value": "fine-grained LatentMoE: 896 routed experts operate in a compact latent width 3584 (0.5× hidden); 2 full-width shared experts", "provenance": "stated", "ref": "§2.3, Table 1 (Latent MoE Dimension 3584)" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.1 (layerwise hybrid of KDA linear attention and Gated MLA)" }, "attention_layer_pattern": { "value": "per block: 3 KDA layers then 1 Gated MLA layer, repeated throughout the backbone; 69 KDA + 24 MLA layers total, with an additional Gated MLA at the end of the backbone so the final layer is global attention", "provenance": "stated", "ref": "§2.1, Table 1 (Attention-Layer Composition)" }, "depth_mixing": { "value": "attention-residuals", "provenance": "stated", "ref": "§2.2 (AttnRes: each layer attends over prior layer outputs with a learned pseudo-query; Block variant partitions 93 layers into 8 blocks of 12 (partial final block), 9 total blocks counting the embedding)" }, "position_encoding": { "method": { "value": "NoPE", "provenance": "stated", "ref": "§2.1.2, §3.4 (no explicit positional embedding; MLA layers are NoPE; KDA encodes position implicitly through recurrent gating/decay)" }, "rope_base": { "value": null, "provenance": "n/a", "ref": "§3.4 (no RoPE anywhere in the model)" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "§3.4 (no RoPE)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "No positional-encoding modification needed: NoPE extrapolates directly to 1M tokens without RoPE rescaling or interpolation (§3.4). Context is grown via a progressive training curriculum: 8K → 64K during pre-training, 256K → 1M during cooldown.", "ref": "§3.4" }, "factor": { "value": null, "provenance": "n/a", "ref": "§3.4 (no rescaling factor applies; no RoPE)" }, "target_length": { "value": 1000000, "provenance": "stated", "ref": "§3.4 (supports a context window of up to 1 million tokens)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.1.1 (head-wise RMSNorm on KDA output), §2.2 (RMSNorm on AttnRes keys), §2.3.1 (RMSNorm in LatentMoE), §2.4 (MoonViT-V2)" }, "placement": { "value": null, "provenance": "unknown", "note": "report does not describe pre/post norm placement" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "QK-norm is a softmax-attention concept; KDA's L2-normalized queries/keys (§2.1.1) are part of the linear-attention parameterization, not evidence of QK-norm. Gated MLA layers are silent on QK-norm." } }, "activation": { "value": "SiTU-GLU", "provenance": "stated", "ref": "§2.3.2, Table 1" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: architecture described in full detail (§2); no designed sink mechanism. Emergent sinks in linear-attention models would be prose, not schema." }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no attention-logit softcapping described. SiTU-GLU applies tanh soft-capping to FFN activations only (§2.3.2), not attention logits." }, "other": [ { "value": "Lower-bounded KDA decay: scaled-sigmoid log-decay with g_min = −5 keeps cumulative decay within BF16 range, eliminating the position-pair diagonal path (all tiles dense Tensor-Core)", "provenance": "stated", "ref": "§2.1.1" }, { "value": "RMSNorm between routed-expert aggregation and up-projection in Stable LatentMoE", "provenance": "stated", "ref": "§2.3.1" }, { "value": "SiTU-GLU bounded activations (β1 = 4 gate, β2 = 25 up; |f(x)| ≤ 100)", "provenance": "stated", "ref": "§2.3.2" }, { "value": "FP32 attention output during training to correct flash-attention rounding bias", "provenance": "stated", "ref": "§2.1.2" }, { "value": "Per-Head Muon: Newton–Schulz orthogonalization applied per head block of attention projections", "provenance": "stated", "ref": "§2.5" }, { "value": "Weight clipping (mechanism carried over from Kimi K2)", "provenance": "stated", "ref": "§3.3" } ] } }, "context": { "trained_length": { "value": 1000000, "provenance": "stated", "ref": "§3.4 (window grows 8K → 64K pre-training, 256K → 1M cooldown), Table 1 (Training Context Length 1M)" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "§3.4 (supports a context window of up to 1 million tokens)" }, "extension_method": { "value": "progressive context-extension curriculum (8K → 64K pre-training; 256K → 1M cooldown); no positional-encoding modification required (NoPE)", "provenance": "stated", "ref": "§3.4" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "report never describes the tokenizer; only the 160K vocab size appears (Table 1)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "no total pre-training token count disclosed; the report retunes the tokens-per-parameter ratio via scaling laws (§3.2) but prints no figure" }, "data_composition": { "value": "curated corpus over four text domains — Web Text, Code, Mathematics, Knowledge — plus a large-scale vision corpus (captions, interleaved image–text documents, OCR, perception, video, visual coding); knowledge and mathematics corpora rephrased with style/perspective-diverse prompting and fidelity verification (Kimi K2 recipe); substantial programmatic multimodal data coupling code with rendered visuals (SVG, 3D assets, Webpage, Game, CAD schematics); coordinate supervision in absolute and normalized formats; pipelines build on Kimi K2/K2.5", "provenance": "stated", "ref": "§3.1" }, "curriculum": { "value": "native multimodal training from the start (vision and language jointly optimized, interleaved tokens under one next-token objective); progressive context extension (8K → 64K pre-training; 256K → 1M cooldown); long-context data upsampled and synthesized (permuted/concatenated multimodal documents) so tasks require full-context attention", "provenance": "stated", "ref": "§3.3, §3.4" }, "optimizer": { "value": "Per-Head Muon (Muon with per-head Newton–Schulz orthogonalization for attention projections) with weight clipping (from Kimi K2)", "provenance": "stated", "ref": "§2.5, §3.3" }, "lr_schedule": { "value": "cosine decay with 1% linear warmup (scaling-law study favored cosine over Warmup Stable Decay under per-schedule optimal hyperparameters); weight decay 0.1 throughout", "provenance": "stated", "ref": "§3.2, §3.3" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "batch size values not disclosed (scaling-law study retuned them, §3.2)" }, "precision": { "value": null, "provenance": "unknown", "note": "master training precision not stated; disclosed: attention output kept FP32 during training (§2.1.2), most activations stored block-wise FP8 with offload (§5.2.2), MXFP4 weights/MXFP8 activations under QAT from SFT onward (§4.1.4)" }, "parallelism": { "value": "Pipeline Parallelism with virtual stages, Expert Parallelism (MoonEP), ZeRO-1 Data Parallelism, Pipeline ZeRO-2 gradient sharding (CPU offload), and Context Parallelism (KDA Context Parallelism / KCP); all-to-all dispatch overlapped with computation; P2P Muon orthogonalization", "provenance": "stated", "ref": "§5.2" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed (only 'a few hundred GPUs' per 1M-context RL experiment, §5.3.1; H20 appears in evaluation configs, not training)" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1.1 (three-stage pipeline begins with SFT; expanded agentic SFT dataset synthesized with prior Kimi models plus multi-stage verification and human-in-the-loop annotation; XTML chat template; QAT from the SFT stage onward)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§4.1.2 (policy optimization follows the Kimi K2.5 algorithm with per-token regularization for stale-data tolerance; agentic generative reward model (GRM) with tournament-style group reward and binary comparisons, verbosity-budgeted; report never names RLHF/PPO/DPO/GRPO)" }, "reasoning_training": { "value": "multi-effort RL: three domain experts (general, general agents, coding agents) × three reasoning-effort levels (low/high/max) with per-problem token-budget control and τ annealing; long-horizon agentic RL at up to 1M context (partial rollouts, persistent sandbox states); verifiable environments: knowledge-graph-guided task synthesis, autonomous execution tasks (AET), kernel optimization, web development, personal assistant workflows, vision-in-the-loop tool use", "provenance": "stated", "ref": "§4.1.2, §4.2" }, "distillation": { "value": "Multi-Teacher On-Policy Distillation (MOPD): the nine domain × effort expert policies are consolidated into the unified model via per-token on-policy distillation rewards (clipped log-ratio), with effort-conditioned training", "provenance": "stated", "ref": "§4.1.3" } }, "modality": { "type": { "value": "multimodal", "provenance": "stated", "ref": "§2.4 (text, images, and videos processed by a single shared backbone within one context)" }, "attachment": { "value": "native", "provenance": "stated", "ref": "§2.4 ('natively multimodal… no post-hoc modality-alignment stage'); MoonViT-V2 (27-layer ViT, ~0.4B/401M params, patch 14, 12 heads) trained from scratch with next-token prediction + lightweight MLP projector; pixel-shuffle 2×2 token downsampling (up to 3584×3584 px); factorized intra-frame spatial / inter-frame temporal attention with temporal pooling; language and vision jointly optimized from the start of training (§3.3)" } }, "efficiency": { "kv_cache": { "value": "hybrid cache: KDA keeps a fixed-size recurrent state (d_k × d_v per head) instead of a growing KV cache; Gated MLA keeps the low-rank latent KV cache (per-token latent, reconstructed keys/values); both managed jointly in a unified paged prefix-cache pool with KDA checkpoints at 512-token hash boundaries", "provenance": "stated", "ref": "§5.1, §5.4.1" }, "quantization": { "value": "MXFP4 MoE expert weights with MXFP8 activations at deployment (QAT throughout SFT and RL); all non-expert components (attention projections, latent MoE projections, shared experts, routers) remain in higher precision", "provenance": "stated", "ref": "§4.1.4" }, "speculative_dedup": { "value": "EAGLE-3-style draft model fine-tuned from the pre-trained MTP layer (draft mirrors one backbone block; 7-step unroll; LK loss directly maximizes acceptance rate; draft input fuses features from AttnRes blocks 1, 4, and final)", "provenance": "stated", "ref": "§4.1.4" }, "serving": { "value": "FlashKDA kernels (training/prefill; auto-dispatched flash-linear-attention backend); SM-level intra-device context parallelism; KDA-aware prefix cache with fine-grained (512-token) hash blocks and sparse KDA checkpoints; WarpDecode-based token-centric MoE decoding kernels; cache-aware affinity scheduling with consistent-hashing failover; budget-based admission control; prefill/decode disaggregation", "provenance": "stated", "ref": "§5.1, §5.4" } }, "evaluation": { "benchmarks": [ { "name": { "value": "GPQA Diamond", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 93.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "CritPt", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 23.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 74.7, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HLE-Full (w/o tools / w/ tools)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "43.5 / 56.0", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "DeepSWE", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 67.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ProgramBench", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 77.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Terminal-Bench 2.1", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "FrontierSWE", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 81.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SWE-Marathon", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 42.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SciCode", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 58.7, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 91.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "DeepSearchQA (F1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 95.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ResearchRubrics", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 76.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MCPMark-Verified", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 94.5, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MCP-Atlas", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 84.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AutomationBench", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 30.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "SpreadsheetBench 2", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 34.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "OSWorld-Verified", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 84.8, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Harvey Lab-AA (criterion pass rate)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 94.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "CorpFin v2", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 71.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "WorldVQA ForceAnswer", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 51.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "OmniDocBench", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 91.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Video-MME (w/ sub)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 90.0, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMVU", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 82.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMMU-Pro (w/o tools / w/ Python)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "81.6 / 83.4", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Math-Vision (w/o tools / w/ Python)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "94.3 / 97.8", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "ZeroBench-main pass@5 (w/o tools / w/ Python)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": "23.0 / 41.0", "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "Artificial Analysis Intelligence Index v4.1", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 57.1, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Vals Index", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 74.7, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "WebDev Arena (Elo)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 1678, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } } ] }, "contributions": { "claimed": [ { "text": { "value": "First open 3T-class model: a 2.8T-parameter native multimodal MoE with 104B activated parameters and a 1M-token context window", "provenance": "stated", "ref": "§1, §8" }, "ref": { "value": "§1, §8", "provenance": "stated", "ref": "§1, §8" } }, { "text": { "value": "Hybrid attention: three KDA layers (delta-rule linear attention with channel-wise forget gate, lower-bounded decay, full-rank output gating) per one Gated MLA layer, giving efficient long-sequence mixing with periodic global interaction", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Attention Residuals (AttnRes) with a Block variant: selective, learned attention over prior layer outputs along the depth axis", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "Stable LatentMoE: 896 routed experts with 16 active per token (56× sparsity), stabilized by RMSNorm, SiTU-GLU, and Quantile Balancing", "provenance": "stated", "ref": "§2.3" }, "ref": { "value": "§2.3", "provenance": "stated", "ref": "§2.3" } }, { "text": { "value": "MoonViT-V2 vision encoder trained entirely from scratch with next-token prediction, matching SigLIP-initialized baselines without contrastive pre-training", "provenance": "stated", "ref": "§2.4" }, "ref": { "value": "§2.4", "provenance": "stated", "ref": "§2.4" } }, { "text": { "value": "Per-Head Muon optimizer: per-head Newton–Schulz orthogonalization of attention projection momentum", "provenance": "stated", "ref": "§2.5" }, "ref": { "value": "§2.5", "provenance": "stated", "ref": "§2.5" } }, { "text": { "value": "RL across general, agentic, and coding domains at three reasoning-effort levels, consolidated into a unified model via Multi-Teacher On-Policy Distillation", "provenance": "stated", "ref": "§4.1" }, "ref": { "value": "§4.1", "provenance": "stated", "ref": "§4.1" } }, { "text": { "value": "Infrastructure: FlashKDA kernels and KDA Context Parallelism, MoonEP perfectly balanced expert-parallel training, million-token agentic RL with AgentENV resumable sandboxes, and KDA-aware prefix caching for serving", "provenance": "stated", "ref": "§5" }, "ref": { "value": "§5", "provenance": "stated", "ref": "§5" } }, { "text": { "value": "Approximately 2.5× improvement in overall scaling efficiency over Kimi K2", "provenance": "stated", "ref": "Abstract, §3.2" }, "ref": { "value": "Abstract, §3.2", "provenance": "stated", "ref": "Abstract, §3.2" } } ], "assessment": { "value": "The block is a deliberate three-axis hybrid (token/depth/channel). vs DeepSeek-V3, the closest atlas peer: same MLA + shared/routed MoE lineage, but K3 replaces the all-softmax-attention backbone with a 3:1 KDA/MLA hybrid, scales routing to 896/16 experts in a latent width (vs 256/8 at full width), and replaces DeepSeek-V3's sign-based bias updates with quantile-based QB. The lower-bounded decay parameterization is a numerical-stability refinement of the delta-rule recurrence family, not a new mixing family. The claimed 2.5× scaling-efficiency gain over Kimi K2 is internal to the report and not independently verifiable. From-scratch vision training with a next-token objective (no contrastive initialization) is the first such claim in the atlas; the report's own evidence is that it matches the SigLIP-initialized baseline with better gradient stability. Depth-wise AttnRes is a genuinely new axis for the atlas (no peer describes attention over layers).", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, mixtral, jamba, llama-3.1, phi-3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "92/93 layers routed MoE (1 dense layer, Table 1)" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "69 KDA (linear-attention family) + 24 MLA (full attention) layers; KDA share 69/93 = 74% < 80% threshold; deliberate 3:1 per-block alternation" }, "modality": { "value": "multimodal", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data composition described but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "2780B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a general frontier model ('open frontier intelligence'); broad benchmark suite across long-horizon coding, agentic, knowledge, reasoning, and vision. Secondary intents: long-context (1M) and agentic-RL emphasis, noted in prose." } }, "prose": { "design_overview": { "value": "Kimi K3 is a 93-layer, 2.78T-parameter sparse MoE (104.2B active per token) that mixes information along three axes at once. For sequence length, each block interleaves three Kimi Delta Attention layers — a delta-rule linear attention with a channel-wise forget gate, lower-bounded (g_min = −5) decay that keeps chunkwise computation inside the BF16 range, and full-rank output gating — with one Gated MLA layer (DeepSeek-V2-style latent KV, NoPE, full-rank gate) for unrestricted global interaction; 23 blocks plus a final Gated MLA give 69 KDA / 24 MLA layers. For depth, Attention Residuals let each layer attend over prior layer outputs via a learned pseudo-query, partitioned into 8 blocks of 12 layers (plus a partial final block) to bound memory. For width, Stable LatentMoE routes each token through 16 of 896 fine-grained experts operating in a 3584-wide latent space (two full-width shared experts carry common transformations), stabilized by RMSNorm before the up-projection, the bounded SiTU-GLU activation, and Quantile Balancing, an auxiliary-loss-free bias update that sets each expert's bias from the quantile of router-score margins. Native vision arrives through MoonViT-V2, a 27-layer ViT trained from scratch with the next-token objective and projected into the shared embedding space; the 1M context is reached by progressive curriculum (8K→64K pre-training, 256K→1M cooldown) with no positional-encoding changes — the model is fully NoPE, with position carried implicitly by KDA's decay. Post-training runs SFT → domain/effort-specialized RL (general, agentic, coding × low/high/max) → multi-teacher on-policy distillation into one model, with MXFP4 QAT from SFT onward.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Multi-head Latent Attention (MLA) — latent KV compression retained in the Gated MLA layers", "provenance": "stated", "ref": "§2.1.2 (cites DeepSeek-V2 [28])" }, "origin": { "value": "DeepSeek-V2", "provenance": "stated", "ref": "§2.1.2" } }, { "mechanism": { "value": "Shared + fine-grained routed expert organization (Stable LatentMoE follows DeepSeekMoE structure)", "provenance": "stated", "ref": "§2.3 (cites DeepSeekMoE [22], DeepSeek-V3 [27])" }, "origin": { "value": "DeepSeek-V3 / DeepSeekMoE", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "Auxiliary-loss-free bias-based load balancing (QB replaces the fixed-step sign update with a quantile update)", "provenance": "stated", "ref": "§2.3.3 (cites DeepSeek-V3 [27])" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.3.3" } }, { "mechanism": { "value": "LatentMoE — routed experts operating in a compact latent space", "provenance": "stated", "ref": "§2.3 (cites LatentMoE [31])" }, "origin": { "value": "LatentMoE (not in atlas)", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "EAGLE-3-style speculative decoding — draft model fine-tuned from the MTP layer", "provenance": "stated", "ref": "§4.1.4 (cites EAGLE-3 [70])" }, "origin": { "value": "EAGLE-3 (not in atlas)", "provenance": "stated", "ref": "§4.1.4" } } ], "influence_out": [], "notable_omissions": { "value": [ "No total pre-training token count disclosed (tokens-per-parameter ratio retuned but never printed)", "No training compute (FLOPs/GPU-hours) or training hardware disclosed", "Master training precision (BF16/FP8) not stated", "Tokenizer algorithm not described (only the 160K vocab size appears in Table 1)", "Batch size and peak learning rate values not disclosed", "No license named for the weights release", "Normalization placement (pre/post) not described", "Input/output embedding tying not disclosed", "Head dimension and KDA state dimensions (d_k, d_v) not disclosed", "The single 'dense layer' counted in Table 1 is never explained in the text", "RL algorithm details deferred to Kimi K2.5 ('follows the algorithm in Kimi K2.5') without naming the method", "Third-party benchmark scores (Table 5, several Table 2 rows) are cited from Artificial Analysis / Vals AI / official leaderboards rather than measured in-house" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "moonshot-kimi-k3.json" }, { "slug": { "value": "moonshot-kimi-linear-48b-a3b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2510.26692", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/moonshot-kimi-linear-48b-a3b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Kimi Linear", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "Kimi Team (Moonshot AI)", "provenance": "stated", "ref": "Title byline; Appendix A ('The project is developed at Moonshot AI'); header (github.com/MoonshotAI/Kimi-Linear)" }, "release_date": { "value": "2025-11-01", "provenance": "stated", "ref": "Title page: 'arXiv:2510.26692v2 [cs.CL] 1 Nov 2025'" }, "report_url": { "value": "https://arxiv.org/abs/2510.26692", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'we open-source the KDA kernel and vLLM implementations, and release the pre-trained and instruction-tuned model checkpoints'; fn 2 (huggingface.co/moonshotai/Kimi-Linear-48B-A3B-Instruct)" } }, "scale": { "total_params": { "value": 48, "provenance": "stated", "ref": "Abstract, §5.4 ('48 billion total parameters')" }, "active_params_per_token": { "value": 3, "provenance": "stated", "ref": "Abstract, §5.4 ('3 billion active parameters per forward pass')" }, "layers": { "value": null, "provenance": "unknown", "note": "layer count of the released 48B-A3B model never printed; only scaling-law models (16–24 layers, Table 2) and the 16-layer ablation are specified" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "hidden dimension of the 48B model not stated; scaling-law models use 1216–1776 (Table 2)" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "expert hidden dimension not disclosed" }, "ffn_ratio": { "value": null, "provenance": "unknown" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "head count for the 48B model not stated (Figure 7 benchmark uses 'the same number of layers and attention heads' as baselines but prints no value)" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§4 (MLA layers cache a low-rank latent — no KV heads; KDA layers keep a recurrent state — no KV cache; NoPE MLA converts to pure MQA at inference, still no KV heads)" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "§4 ('dk, dv represent the key and value head dimensions, which are set to 128 for all experiments')", "note": "per-head key/value dimension of the KDA layers (q also lives in R^dk); the MLA latent dimension is not separately disclosed" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "vocabulary size never stated" }, "embedding_tied": { "value": null, "provenance": "unknown" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "the first layer is implemented as a dense layer without MoE (§5.4); all remaining layers use routed MoE — MoE share ≈ 100% of non-first layers ≥ 80%" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "§5.4 ('Each model activates 8 out of 256 experts')" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§5.4 ('activates 8 out of 256 experts')" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "§5.4 ('including one shared expert')" }, "routing": { "value": null, "provenance": "unknown", "note": "router gating function not described" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no load-balancing mechanism described" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "expert organization not described; only counts and 'MoE sparsity is increased to 32' (§5.4)" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§4 (layerwise hybrid of KDA linear attention and full MLA global attention)" }, "attention_layer_pattern": { "value": "uniform 3:1 interleaving — 3 KDA (linear-attention) layers per 1 full MLA layer, repeated through the stack (N = 3, Figure 3); the first layer is dense (no MoE); 3:1 chosen by hybrid-ratio ablation", "provenance": "stated", "ref": "§4, §5.2, Table 1 (hybrid-ratio ablation: 3:1 best; 0:1, 1:1, 7:1, 15:1 worse)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no learned attention over prior layer outputs described (§4)" }, "position_encoding": { "method": { "value": "NoPE", "provenance": "stated", "ref": "§4 ('we apply NoPE to all full attention (MLA) layers'); §6.1 (KDA's data-dependent decay acts as a learnable multiplicative positional encoding, so no explicit PE anywhere)" }, "rope_base": { "value": null, "provenance": "n/a", "ref": "§4 (no RoPE in the model; 'Kimi Linear (RoPE)' is an ablation baseline only, §5.4)" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "§4 (no RoPE)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "No positional-encoding modification needed: NoPE 'obviates the need for RoPE parameter adjustments, such as frequency base tuning or methods like YaRN' (§4). Long context is reached via the Kimi K2 annealing + long-context activation phase (§5.4.1).", "ref": "§4, §5.4.1" }, "factor": { "value": null, "provenance": "n/a", "ref": "§4 (no rescaling factor applies; no RoPE)" }, "target_length": { "value": 1000000, "provenance": "stated", "ref": "§5.4.1 ('the final checkpoint supports a context length of up to 1 million tokens')" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§4 (head-wise RMSNorm before the output projection)" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "L2Norm on KDA q/k is linear-attention parameterization for eigenvalue stability, not QK-normalisation on global-attention layers (schema rule); MLA layers are silent on QK-norm — absence rule" } }, "activation": { "value": "Swish", "provenance": "stated", "ref": "§4 (q, k, v = Swish(ShortConv(·))); MoE FFN activation not described" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens; the low-rank Sigmoid output gate is reported to 'alleviate the Attention Sink' (§4) — a mitigation, not a designed sink" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no attention-logit softcapping described" }, "other": [ { "value": "L2 normalization of q/k after Swish+ShortConv to ensure eigenvalue stability (as suggested by [112])", "provenance": "stated", "ref": "§4" }, { "value": "Head-wise RMSNorm plus a low-rank data-dependent Sigmoid output gate, reported to match full-rank gating while alleviating the attention sink", "provenance": "stated", "ref": "§4" }, { "value": "Channel-wise (per-dimension) forget gate Diag(αt) via a low-rank projection with a bounded decay function f(·), giving fine-grained memory control (unlike GDN's scalar gate)", "provenance": "stated", "ref": "§3, §4" } ] } }, "context": { "trained_length": { "value": 4096, "provenance": "stated", "ref": "§5.4.1 ('All models are pretrained using a 4,096-token context window')" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "§5.4.1 ('the final checkpoint supports a context length of up to 1 million tokens')" }, "extension_method": { "value": "same annealing schedule and long-context activation phase established in Kimi K2; no RoPE/YaRN adjustments required because the model is NoPE", "provenance": "stated", "ref": "§5.4.1, §4" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer never described; vocabulary size never stated" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 5.7, "provenance": "stated", "ref": "§5.4.1 ('an expanded total of 5.7 trillion tokens to match the pretraining tokens of Moonlight')", "note": "the released checkpoint is the 5.7T run; all matched-scale comparisons against MLA/GDN-H used 1.4T (Tables 3–5)" }, "data_composition": { "value": "1.4T tokens sampled from the Kimi K2 pretraining corpus; the 5.7T run follows the same procedure on the K2 corpus", "provenance": "stated", "ref": "§5.4.1" }, "curriculum": { "value": "same annealing schedule and long-context activation phase established in Kimi K2", "provenance": "stated", "ref": "§5.4.1" }, "optimizer": { "value": "MuonClip (inherited from Kimi K2)", "provenance": "stated", "ref": "§5.4.1" }, "lr_schedule": { "value": "WSD learning-rate schedule; learning rate 1.1 × 10⁻³", "provenance": "stated", "ref": "§5.4.1" }, "batch_schedule": { "value": "global batch size fixed at 32 million tokens", "provenance": "stated", "ref": "§5.4.1" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not stated (RL section notes a 'precision mismatch between training and inference engines' without naming precisions, §5.4.2)" }, "parallelism": { "value": null, "provenance": "unknown" }, "hardware": { "value": null, "provenance": "unknown" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed; scaling-law study reports only relative ~1.16× compute efficiency vs MLA (Figure 5)" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§5.4.2 (multi-stage SFT: broad instruction-following data first, then scheduled reasoning-intensive data; dataset extends the Kimi K2 SFT data)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§5.4.2 (same RL algorithm as Kimi K1.5, with truncated importance sampling, dynamically adjusted KL penalty and mini-batch size; RLVR on math/code/STEM prompt sets; no RLHF/PPO/DPO/GRPO named)" }, "reasoning_training": { "value": "RLVR on mathematics, code and STEM prompt sets pre-selected at moderate difficulty; truncated importance sampling to counter train/inference precision mismatch; dynamic KL penalty and mini-batch adjustment to avoid entropy collapse; PTX loss for general-capability retention", "provenance": "stated", "ref": "§5.4.2" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "SFT data extends the Kimi K2 SFT dataset (teacher-generated data, not distillation per schema rule — noted in prose)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: no multimodal components described; all evaluation benchmarks are text-only" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": "hybrid cache: KDA layers keep a fixed-size recurrent state (dk × dv = 128 × 128 per head, independent of sequence length) instead of a growing KV cache; MLA layers (1 in 4) keep the low-rank latent KV cache and can be converted to pure MQA at inference under NoPE — up to 75% KV-cache reduction vs full MLA, up to 6× decoding throughput at 1M context", "provenance": "derived", "note": "from attention_layer_pattern (§4) plus stated efficiency claims (Abstract, §1, §6.3)" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "no quantization formats discussed" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding described" }, "serving": { "value": "open-source KDA kernels integrated with vLLM, drop-in compatible with existing caching and scheduling interfaces; prefill uses the FLOP-intensive chunkwise kernel, autoregressive generation switches to the recurrent kernel; hybrid I/O-bounded decoding approaches a 3:1 efficiency ratio vs full attention", "provenance": "stated", "ref": "Abstract, §3.1, §6.3" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro (4k)", "provenance": "stated", "ref": "Figure 1, Table 3" }, "value": { "value": 51.0, "provenance": "stated", "ref": "Figure 1, Table 3" }, "ref": { "value": "Figure 1, Table 3", "provenance": "stated", "ref": "Figure 1, Table 3" } }, { "name": { "value": "RULER (128k)", "provenance": "stated", "ref": "Figure 1, Table 5" }, "value": { "value": 84.3, "provenance": "stated", "ref": "Figure 1, Table 5" }, "ref": { "value": "Figure 1, Table 5", "provenance": "stated", "ref": "Figure 1, Table 5" } }, { "name": { "value": "HellaSwag (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 82.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ARC-Challenge (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 67.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Winogrande (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 78.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BBH (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 72.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 73.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "TriviaQA (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 71.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GSM8K (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 83.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MATH (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 54.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "EvalPlus (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 60.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "CRUXEval-I-cot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 56.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "CRUXEval-O-cot (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 62.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "CEval (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 79.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "CMMLU (Base)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 80.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BBH (Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 69.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU (Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 77.0, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Pro (Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 67.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Redux (Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 80.3, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA-Diamond (Avg@8, Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 62.1, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LiveBench (Pass@1, Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 45.2, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "AIME 2025 (Avg@64, Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 21.3, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MATH500 (Acc., Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 81.2, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HMMT 2025 (Avg@32, Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 12.5, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "PolyMath-en (Avg@4, Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 43.6, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LiveCodeBench v6 (Pass@1, Instruct)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 26.0, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MRCR (128k)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 29.6, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HELMET-ICL (128k)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 90.0, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LongBench V2 (128k)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 35.0, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Frames (128k)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 58.8, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "RepoQA (128k)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 68.5, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Long Code Arena (128k)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 37.1, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "RULER@1M (Instruct, 5.7T)", "provenance": "stated", "ref": "Appendix D, Table 9" }, "value": { "value": 94.8, "provenance": "stated", "ref": "Appendix D, Table 9" }, "ref": { "value": "Appendix D, Table 9", "provenance": "stated", "ref": "Appendix D, Table 9" } }, { "name": { "value": "MMLU-Pro (Base, 5.7T)", "provenance": "stated", "ref": "Appendix D, Table 8" }, "value": { "value": 54.8, "provenance": "stated", "ref": "Appendix D, Table 8" }, "ref": { "value": "Appendix D, Table 8", "provenance": "stated", "ref": "Appendix D, Table 8" } }, { "name": { "value": "GPQA-Diamond (Avg@8, Instruct, 5.7T)", "provenance": "stated", "ref": "Appendix D, Table 9" }, "value": { "value": 71.7, "provenance": "stated", "ref": "Appendix D, Table 9" }, "ref": { "value": "Appendix D, Table 9", "provenance": "stated", "ref": "Appendix D, Table 9" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Kimi Delta Attention (KDA): a gated-delta-rule linear attention with channel-wise (fine-grained) gating, extending Gated DeltaNet for more effective use of finite-state RNN memory", "provenance": "stated", "ref": "Abstract, §3" }, "ref": { "value": "Abstract, §3", "provenance": "stated", "ref": "Abstract, §3" } }, { "text": { "value": "A bespoke chunkwise-parallel algorithm based on a constrained Diagonal-Plus-Low-Rank (DPLR) transition, roughly 2× faster than general DPLR kernels (≈100% operator-efficiency gain) while staying consistent with the classical delta rule", "provenance": "stated", "ref": "Abstract, §3.2, §6.2" }, "ref": { "value": "Abstract, §3.2, §6.2", "provenance": "stated", "ref": "Abstract, §3.2, §6.2" } }, { "text": { "value": "The first hybrid linear-attention architecture to outperform full attention under fair comparisons across short-context, long-context and RL scaling regimes (1.4T-token matched runs)", "provenance": "stated", "ref": "Abstract, §1, §5" }, "ref": { "value": "Abstract, §1, §5", "provenance": "stated", "ref": "Abstract, §1, §5" } }, { "text": { "value": "A 3:1 KDA-to-MLA layerwise hybrid that reduces memory and KV-cache usage by up to 75% and achieves up to 6× decoding throughput at 1M context, positioned as a drop-in replacement for full-attention architectures", "provenance": "stated", "ref": "Abstract, §1, §4" }, "ref": { "value": "Abstract, §1, §4", "provenance": "stated", "ref": "Abstract, §1, §4" } }, { "text": { "value": "Open release of KDA kernels (flash-linear-attention), vLLM integration, and pre-trained plus instruction-tuned checkpoints, drop-in compatible with existing caching/scheduling interfaces", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } } ], "assessment": { "value": "KDA is a synthesis inside the delta-rule family rather than a new mixing family: Gated DeltaNet's recurrence with GLA-style channel-wise gates and DeltaNet's Householder/delta update, plus a genuinely useful kernel contribution (the a=b=k-constrained DPLR chunkwise algorithm, ~2× over general DPLR). As a system, Kimi Linear follows the inter-layer hybrid paradigm already present in the atlas via ai21-jamba (Transformer+Mamba 52B), but it is the first delta-rule-based (KDA) hybrid in the atlas and the first entry whose linear component beats the full-attention baseline under matched training. The NoPE-global-attention choice is shared with recent hybrids (Falcon-H, SwanGPT, and the report's own ablation vs RoPE) and is well motivated. The 'outperforms full attention' claim is internally consistent (identical recipe, 1.4T matched runs) but validated only at 48B/3B scale. The KDA mechanism is the direct ancestor of the KDA layers in moonshot-kimi-k3, whose report refines the same recurrence; the K3 report in this atlas does not credit Kimi Linear by name, but the mechanism is unambiguous.", "provenance": "inferred", "note": "comparison against atlas entries (ai21-jamba, moonshot-kimi-k3, deepseek-v3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all layers except the first dense layer use routed MoE (§5.4) — ≥ 80% layer-majority rule" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "uniform 3:1 KDA-to-MLA interleaving: KDA (linear-attention family) share = 75% < 80% threshold; deliberate alternating per-block pattern (§4)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (checkpoints + kernels + vLLM integration released); pretraining corpus is the Kimi K2 corpus — described but not released" }, "scale_class": { "value": "large", "provenance": "derived", "note": "48B total params ∈ [10B, 100B)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a drop-in general-purpose replacement for full attention ('superior performance and efficiency') aimed at the demands of agentic intelligence and test-time scaling; evaluated broadly (general, math/code, Chinese, long-context). Secondary intents: long-context (1M) and decoding efficiency, noted in prose." } }, "prose": { "design_overview": { "value": "Kimi Linear is a 48B-parameter sparse MoE (3B active) that interleaves token mixing at a uniform 3:1 ratio: three Kimi Delta Attention (KDA) layers for every one full MLA layer. KDA is a gated-delta-rule linear attention — St = (I − βt kt ktᵀ) Diag(αt) S_{t−1} + βt kt vtᵀ — that fuses Gated DeltaNet's delta-rule recurrence with GLA-style channel-wise forgetting: each of the 128 key/value dimensions has its own decay αt, parameterized by a low-rank projection through a bounded decay function, giving fine-grained control of the fixed 128×128 per-head recurrent state. Queries/keys pass through ShortConv + Swish + L2Norm for eigenvalue stability, and the head output gets RMSNorm plus a low-rank data-dependent Sigmoid gate (which the report ties to alleviating attention sink). The kernel story is the second half of the contribution: by binding the DPLR factors to k (a = b = k), the chunkwise algorithm avoids two secondary-chunking steps and three matrix multiplications, running ~2× faster than general DPLR while remaining mathematically the classical delta rule. The whole model is NoPE — MLA layers carry no positional encoding and KDA's learnable decay acts as a multiplicative positional encoding — which also lets MLA convert to pure MQA at inference. The MoE (8/256 experts plus one shared, first layer dense) and the entire recipe (MuonClip, WSD, K2 corpus, annealing/long-context activation) are inherited from Moonlight and Kimi K2. Released with open kernels, vLLM integration, and checkpoints; the same KDA recurrence later becomes the linear-attention layer of Kimi K3, the atlas's next family record.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Gated DeltaNet (GDN) — KDA extends its gated delta rule with fine-grained channel-wise gating", "provenance": "stated", "ref": "Abstract, §3 (cites [111])" }, "origin": { "value": "Gated Delta Networks", "provenance": "stated", "ref": "Abstract, §3" } }, { "mechanism": { "value": "Channel-wise (diagonal) gating from Gated Linear Attention (GLA)", "provenance": "stated", "ref": "§1, §6.2 (cites GLA [114])" }, "origin": { "value": "GLA", "provenance": "stated", "ref": "§1, §6.2" } }, { "mechanism": { "value": "Delta rule / fast-weight programming from DeltaNet (Householder-style rank-1 updates, WY representation)", "provenance": "stated", "ref": "§2.2, §3.1 (cites [84, 112])" }, "origin": { "value": "DeltaNet", "provenance": "stated", "ref": "§2.2, §3.1" } }, { "mechanism": { "value": "Backbone architecture following Moonlight (incl. Muon optimizer for scaling-law experiments)", "provenance": "stated", "ref": "§4 ('The main backbone of our model architecture follows Moonlight [62]'), §5.3" }, "origin": { "value": "Moonlight (Moonshot)", "provenance": "stated", "ref": "§4" } }, { "mechanism": { "value": "Pretraining recipe from Kimi K2: MuonClip optimizer, WSD schedule, annealing + long-context activation phase, K2 pretraining corpus", "provenance": "stated", "ref": "§5.4.1 (cites Kimi K2 [50])" }, "origin": { "value": "Kimi K2", "provenance": "stated", "ref": "§5.4.1" } }, { "mechanism": { "value": "RL algorithm from Kimi K1.5 (with truncated importance sampling and adaptive KL/mini-batch additions)", "provenance": "stated", "ref": "§5.4.2 (cites [95])" }, "origin": { "value": "Kimi K1.5", "provenance": "stated", "ref": "§5.4.2" } }, { "mechanism": { "value": "ShortConv (lightweight depthwise convolution) on attention inputs", "provenance": "stated", "ref": "§4 (following [111])" }, "origin": { "value": "Gated Delta Networks / Mamba-style conv", "provenance": "stated", "ref": "§4" } } ], "influence_out": [ { "model_slug": { "value": "upstage-solar-open-2", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "KDA (Kimi Delta Attention) — gated delta-rule linear attention with fixed-size recurrent state for the 36 linear layers", "provenance": "inferred", "note": "mechanism as stated in upstage-solar-open-2 report's lineage_in" } } ], "notable_omissions": { "value": [ "Layer count, hidden dimension, head count, and expert hidden dimension of the released 48B-A3B model are never printed (only scaling-law sizes 653M–1.7B active, Table 2)", "Vocabulary size and tokenizer not described", "Router gating function and expert load-balancing not described", "MoE FFN activation function not stated (only the attention-path Swish)", "Training precision, parallelism strategy, hardware, and total compute not disclosed", "No license named for released checkpoints/kernels", "Normalization placement (pre/post) not described; embedding tying not disclosed", "MLA latent dimension not disclosed (only KDA's dk = dv = 128)", "Exact decay function f(·) parameterization not specified in the text", "Source extraction: arXiv HTML unavailable (arxiv.org/html 404; ar5iv fatal conversion error), so the PDF was used — inline math and some table cells are partially garbled in the stored text", "Long-context results at 1M reported only for the 5.7T Instruct checkpoint (RULER), not the 1.4T matched runs" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "moonshot-kimi-linear-48b-a3b.json" }, { "slug": { "value": "nanbeige-4-1-3b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2602.13367", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/nanbeige-4-1-3b", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Nanbeige4.1", "provenance": "stated", "ref": "Title ('Nanbeige4.1-3B: A Small General Model that Reasons, Aligns, and Acts')" }, "variants": [], "org": { "value": "Nanbeige LLM Lab, Boss Zhipin", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-02-13", "provenance": "inferred", "note": "report text is undated; arXiv v1 submission date 2026-02-13 per export.arxiv.org API metadata (arXiv:2602.13367)" }, "report_url": { "value": "https://arxiv.org/abs/2602.13367", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('The model checkpoint is available at https://huggingface.co/Nanbeige/Nanbeige4.1-3B'); §1 ('We open-source Nanbeige4.1-3B')" } }, "scale": { "total_params": { "value": 3, "provenance": "stated", "ref": "Abstract ('a unified generalist language model that ... with only 3B parameters')" }, "active_params_per_token": { "value": 3, "provenance": "derived", "note": "dense model: active params per token = total params (3B); no routing described" }, "layers": { "value": null, "provenance": "unknown", "note": "not disclosed (the report is a post-training recipe paper with no architecture section)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: no dimensions disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "head_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not stated" }, "variants": [] }, "block": { "block_type": { "value": "dense", "provenance": "inferred", "note": "the report itself discloses no architecture (post-training recipe paper); task context specifies a 3B dense model, and the report describes initialization from the Nanbeige4-3B-Base (§2.1.1) with no MoE mechanism mentioned anywhere" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§2.1.1", "note": "dense model; no MoE described" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§2.1.1" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§2.1.1" }, "routing": { "value": null, "provenance": "n/a", "ref": "§2.1.1" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§2.1.1" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§2.1.1" } }, "attention_variant": { "value": null, "provenance": "unknown", "note": "no attention mechanism disclosed" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "no attention description at all" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence default: no attention over prior layer outputs described (schema absence rule)" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "not disclosed; context growth is achieved by a training-stage curriculum (§2.1.1), not by a described PE modification" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "context length is extended by adding a third SFT training stage at 256k tokens (32k → 64k → 256k, §2.1.1), i.e. training at the deployed length, with no positional-encoding extension method described" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.1.1", "note": "no rescaling factor applies; no PE extension method described" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§2.1.1", "note": "third SFT stage at 256k tokens; 'we extend the context length ... by introducing a third stage at 256k tokens'" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "placement": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "not disclosed" } }, "activation": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "no architecture disclosure to which the absence rule could apply" }, "softcapping": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "other": { "value": null, "provenance": "unknown", "note": "no block-level stability mechanisms disclosed" } } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§2.1.1", "note": "SFT context curriculum extended to a third stage at 256k tokens (32k → 64k → 256k)" }, "deployed_length": { "value": 262144, "provenance": "derived", "note": "report states the maximum context as 256k tokens ('This longer context window is essential for supporting long-horizon reasoning and multi-turn deep-search planning', §2.4); stored as 256 x 1024 = 262144" }, "extension_method": { "value": "none", "provenance": "inferred", "note": "context growth via an additional SFT training stage at 256k (training at the deployed length); no positional-encoding extension described" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "not described" }, "notes": { "value": null, "provenance": "unknown", "note": "not described" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "no token counts disclosed for any stage" }, "data_composition": { "value": "SFT mixture redesigned with a higher proportion of code-related data and higher ratios of challenging mathematics and general-domain problems; final 256k-stage mixture: code 27%, deep-Search 26%, STEM 23%, tool-use 13%, general domains 10%. Deep-search data: multi-hop QA pairs from Wikipedia entity-relation graphs (temporal-aware head-entity selection, conditional random walks) plus synthesized long-horizon trajectories with turn-level judgment (open-sourced as ToolMind-Web-QA). Code data: judge-system-filtered time-optimal solutions; RL data: point-wise (human preference), pair-wise (strong-weak model comparisons with checklist filtering), code (problem + test cases + time-complexity-optimal solution + optimal complexity label), agentic (1,827 synthesized task-oriented environments with tools and verifiers)", "provenance": "stated", "ref": "§2.1.1, §2.2.1, §2.3.2" }, "curriculum": { "value": "post-training pipeline on the Nanbeige4-3B-Base: extended SFT (three context stages, final 256k) → point-wise RL (GRPO, 8 rollouts per prompt, general reward model) → pair-wise RL (pairwise reward model, swap-consistency regularizer) → two-stage code RL (Stage 1 correctness pass-rate reward; Stage 2 gated time-complexity reward active only at PassRate=1) → lightweight agentic RL; on-policy difficulty filtering in code RL (n=8 rollouts, retain problems with k in [1,5] qualifying rollouts)", "provenance": "stated", "ref": "§2.4, §2.3.2" }, "optimizer": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed (rollout counts of 8 per prompt are RL sampling settings, not batch schedules)" }, "precision": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§2.1.1, §2.4", "note": "extended SFT on Nanbeige4-3B-Base with redesigned data mixture, 256k context stage, and a scaled Solution Refinement / CoT Reconstruction framework" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§2.1.2", "note": "'We then perform GRPO (shao2024deepseekmath) to optimize Nanbeige4.1-3B-SFT'; both point-wise and pair-wise RL stages use GRPO, as do the code and agentic RL stages" }, "reasoning_training": { "value": "cascaded multi-stage RL: point-wise RL with a general reward model (8 rollouts per prompt) suppressing redundancy/repetition (overlong truncation reduced from 5.27% to 0.38% on LiveCodeBench-v6); pair-wise RL with binary win/loss reward vs reference answers, multi-turn dialogue history concatenated into the reward model input; two-stage code RL with correctness pass-rate reward then a gated time-complexity reward (judge system compares generated solutions against reference optimal complexity); lightweight agentic RL for tool-use and search; deep-search training with turn-level judgment (logical soundness, tool-call accuracy, informational gain) and trajectory-level rewards, enabling stable long-horizon interaction of up to 600 tool-call turns", "provenance": "stated", "ref": "§2.1.2, §2.1.3, §2.2.2, §2.3.3, §2.4" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "no teacher-model distillation disclosed; strong LLMs are used to generate and synthesize training data and reward signals (§2.3.2), which per schema §8 is teacher-generated data, not distillation" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes only text-based reasoning, code, and tool interaction; no non-text modality appears" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§1", "note": "text-only; no non-text modality" } }, "efficiency": { "kv_cache": { "value": null, "provenance": "unknown", "note": "no attention/KV-cache disclosure (architecture not described)" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "no quantization format described" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding or multi-token prediction mentioned" }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "no serving optimisations disclosed (the Mindflow framework is used for evaluation only, §2.2.2)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "LiveCodeBench-V6", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 76.9, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LCB-Pro-Easy", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 81.4, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LCB-Pro-Medium", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 28.1, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "AIME 2026 I", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 87.4, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B; report prints 87.40)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HMMT Nov", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 77.92, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "IMO-Answer-Bench", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 53.38, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 83.8, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HLE (Text-only)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 12.6, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B; report prints 12.60)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Arena-Hard-V2", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 73.2, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Multi-Challenge", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 52.21, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "BFCL-V4", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 56.5, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B; report prints 56.50)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Tau2-Bench", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 48.57, "provenance": "stated", "ref": "Table 4 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 4 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GAIA (text-only)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 69.9, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B; report prints 69.90)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 19.12, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BrowseComp-ZH", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 31.83, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HLE (text-only)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 22.29, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SEAL-0", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 41.44, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "xBench-DeepSearch-05", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 75.0, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B; report prints 75.00)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "xBench-DeepSearch-10", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 39.0, "provenance": "stated", "ref": "Table 5 (Nanbeige4.1-3B; report prints 39.00)" }, "ref": { "value": "Table 5 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LeetCode Weekly Contests 484-488 (pass rate)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 85.0, "provenance": "stated", "ref": "Table 6 (Nanbeige4.1-3B; 17 of 20 problems solved)" }, "ref": { "value": "Table 6 (Nanbeige4.1-3B)", "provenance": "stated", "ref": "Table 6" } } ] }, "contributions": { "claimed": [ { "text": { "value": "First open-source small language model (3B) to combine strong agentic behavior, code generation, and general reasoning in a single model", "provenance": "stated", "ref": "Abstract" }, "ref": { "value": "Abstract", "provenance": "stated", "ref": "Abstract" } }, { "text": { "value": "Combination of point-wise and pair-wise reward modeling for reasoning and preference alignment (GRPO with a general reward model, then a pair-wise reward model with a swap-consistency regularizer)", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Complexity-aware rewards in code RL: a gated time-complexity reward activated only when all test cases pass, with a judge system (multi-language sandbox + instruct judge model) comparing generated solutions against the reference optimal complexity bound", "provenance": "stated", "ref": "§2.3" }, "ref": { "value": "§2.3", "provenance": "stated", "ref": "§2.3" } }, { "text": { "value": "Deep-search data synthesis (Wikipedia entity-relation graph random walks with temporal-aware head-entity selection; trajectory synthesis with turn-level judgment on logical soundness, tool-call accuracy, and informational gain) enabling stable long-horizon tool interactions of up to 600 tool-call turns", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "Open-sourced the ToolMind-Web-QA search dataset on HuggingFace", "provenance": "stated", "ref": "§2.2.1" }, "ref": { "value": "§2.2.1", "provenance": "stated", "ref": "§2.2.1" } }, { "text": { "value": "A unified multi-stage training recipe (SFT → point-wise RL → pair-wise RL → two-stage code RL → agentic RL) that keeps a 3B model broadly competent and strongly specialized simultaneously", "provenance": "stated", "ref": "§2.4" }, "ref": { "value": "§2.4", "provenance": "stated", "ref": "§2.4" } }, { "text": { "value": "Deep-search performance at 3B scale that beats specialized search models at tens-of-billions scale and approaches 100B+ general-purpose models (GAIA 69.90, xBench-DeepSearch-05 75.00, SEAL-0 41.44)", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } } ], "assessment": { "value": "A pure post-training contribution on the Nanbeige4-3B base: no architectural novelty is claimed or described (the report has no architecture section at all). The reward-engineering stack — point-wise + pair-wise RL, a gated time-complexity reward, and turn-level vs trajectory-level credit in agentic RL — extends the cascaded-RL line the report cites (Nemotron-Cascade); the gated complexity reward that fires only on fully correct solutions is the most distinctive element. The 600-turn agentic-stability and deep-search claims at 3B scale are strong but not independently verifiable. vs microsoft-phi-4 (the atlas's closest 'small generalist' peer): same small-model-broad-capabilities intent, but Nanbeige4.1 optimizes for agentic deep search and code efficiency rather than instruction-following density; vs weibo-vibethinker-3b: same 3B reasoning scale, but Nanbeige adds long-horizon tool use and alignment stages. The complete absence of architecture disclosure makes cross-atlas block comparison impossible.", "provenance": "inferred", "note": "comparison against atlas entries (microsoft-phi-4, weibo-vibethinker-3b, zai-glm-4-5-air) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "from block_type (inferred dense: task context '3B dense'; report discloses no MoE mechanism and initializes from the dense Nanbeige4-3B-Base)" }, "sequence_mixing": { "value": null, "provenance": "unknown", "note": "the report discloses no attention mechanism whatsoever; no efficient-attention or SSM claim appears, but full-attention cannot be verified from this report, so the tag is unknown rather than guessed" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no non-text modality in the report)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (checkpoint on HuggingFace, Abstract); only the ToolMind-Web-QA search dataset is released, not the training data, so not open-weights-open-data" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "3B: 1B ≤ 3B < 10B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "report frames the model as a unified generalist ('first open-source SLM to achieve such versatility in a single model', Abstract) spanning reasoning, coding, alignment, and long-horizon agentic behavior — broad generalist intent at small scale; not reasoning-specialised, on-device, long-context, or domain-specific per the report's own framing" } }, "prose": { "design_overview": { "value": "Nanbeige4.1-3B is a post-training recipe applied to the Nanbeige4-3B-Base checkpoint; the report discloses no architecture beyond the 3B parameter count and a 256K context, so the block itself (a dense decoder, per family context) is inherited unchanged. The contribution is a cascaded, capability-balanced training stack. SFT is re-mixed toward code and hard math, run through a three-stage context curriculum (32k → 64k → 256k, with the final stage mixing code 27%, deep-search 26%, STEM 23%, tool-use 13%, general 10%), and improved by scaling the Solution Refinement / CoT Reconstruction loop from the prior Nanbeige4-3B-2511 release. General RL then runs GRPO twice: point-wise RL with a general reward model (8 rollouts per prompt) that suppresses redundancy and formatting errors, then pair-wise RL with a binary win/loss reward and a swap-consistency regularizer to remove position bias. Code RL runs in two stages — correctness via a pass-rate reward, then a gated time-complexity reward active only when all tests pass, with a judge system (multi-language sandbox plus an instruct judge model) comparing against reference optimal complexity. Finally a lightweight agentic RL stage, trained on Wikipedia-graph multi-hop QA and turn-judged long trajectories (open-sourced as ToolMind-Web-QA), delivers stable long-horizon search — up to 600 tool-call turns. The result is a 3B model that leads small models on deep-search benchmarks and beats much larger peers on coding and alignment — at the cost of disclosing no architectural detail at all.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Base model and initialization (Nanbeige4.1-3B is built upon the Nanbeige4-3B-Base)", "provenance": "stated", "ref": "§2.1.1" }, "origin": { "value": "Nanbeige4-3B (Boss Zhipin)", "provenance": "stated", "ref": "§2.1.1" } }, { "mechanism": { "value": "Solution Refinement and CoT Reconstruction framework (scaled up from the previous version)", "provenance": "stated", "ref": "§2.1.1" }, "origin": { "value": "Nanbeige4-3B-2511", "provenance": "stated", "ref": "§2.1.1" } }, { "mechanism": { "value": "Point-wise reward modeling / cascaded RL paradigm", "provenance": "stated", "ref": "§2.1.2 (cites Nemotron-Cascade, wang2025nemotron)" }, "origin": { "value": "Nemotron-Cascade (NVIDIA)", "provenance": "stated", "ref": "§2.1.2" } }, { "mechanism": { "value": "Group Relative Policy Optimization (GRPO)", "provenance": "stated", "ref": "§2.1.2 (cites DeepSeekMath, shao2024deepseekmath)" }, "origin": { "value": "DeepSeekMath / DeepSeek-R1", "provenance": "stated", "ref": "§2.1.2" } }, { "mechanism": { "value": "Entity-relation graph random walks and multi-hop QA synthesis for tool-use data (ToolMind framework)", "provenance": "stated", "ref": "§2.2.1 (cites ToolMind, yang2025toolmind)" }, "origin": { "value": "ToolMind (Boss Zhipin)", "provenance": "stated", "ref": "§2.2.1" } }, { "mechanism": { "value": "Unified pairwise RLHF framework with swap-consistency regularizer for position-bias mitigation", "provenance": "stated", "ref": "§2.1.3 (cites xu2025unified)" }, "origin": { "value": "Unified pairwise RLHF framework (Xu et al. 2025)", "provenance": "stated", "ref": "§2.1.3" } } ], "influence_out": [], "notable_omissions": { "value": [ "No architecture disclosure at all: layers, hidden dim, heads, activation, normalization, position encoding, tokenizer, vocab — none described (post-training recipe paper)", "No pre-training details (token counts, data, compute) — the model is initialized from Nanbeige4-3B-Base", "No license named for the release", "No training hardware, optimizer, learning-rate schedule, precision, or parallelism", "No FLOPs or GPU-hours", "No inference-efficiency disclosures (KV cache, quantization, serving)", "No direct measurement of the 600-tool-call-turn ceiling (only the claim)", "General-evaluation settings are minimal (harness details given only for deep-search evals: Mindflow framework, Serper/Jina/E2B tools)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "nanbeige-4-1-3b.json" }, { "slug": { "value": "nanbeige-4-2-3b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://huggingface.co/Nanbeige/Nanbeige4.2-3B/raw/main/Nanbeige42_report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/nanbeige-4-2-3b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Nanbeige4.2", "provenance": "stated", "ref": "Title ('Nanbeige4.2-3B: Unlocking Agentic Capabilities in a Compact Model')" }, "variants": [], "org": { "value": "Nanbeige LLM Lab, Boss Zhipin", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": null, "provenance": "unknown", "note": "report is an undated preprint ('Preprint.' on the title page) hosted on Hugging Face, not arXiv; no date printed anywhere in the document" }, "report_url": { "value": "https://huggingface.co/Nanbeige/Nanbeige4.2-3B/raw/main/Nanbeige42_report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('The model checkpoint is available at https://huggingface.co/Nanbeige/Nanbeige4.2-3B'); §1 ('We open-source Nanbeige4.2-3B')" } }, "scale": { "total_params": { "value": 3, "provenance": "stated", "ref": "Abstract ('a compact general agentic model with 3B non-embedding parameters'); Table 1 and Table 3 list total 4B / non-embedding 3B" }, "active_params_per_token": { "value": 3, "provenance": "derived", "note": "dense model (no MoE disclosed): active = total = 3B non-embedding parameters" }, "layers": { "value": null, "provenance": "unknown", "note": "not disclosed; only that the shared layer stack is reused for a second pass (loop depth 2, §2.1)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed ('we tune the depth of the transformer stack and its hidden width' without printing values, §2.1)" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "ffn_inner_dim and hidden_dim unknown" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "head_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "variants": [] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "Looped Transformer reusing a standard dense Transformer stack; no MoE or sparse mechanism described anywhere (§2.1)" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§2.1", "note": "dense model; no MoE described" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§2.1" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§2.1" }, "routing": { "value": null, "provenance": "n/a", "ref": "§2.1" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§2.1" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§2.1" } }, "attention_variant": { "value": null, "provenance": "unknown", "note": "no attention mechanism disclosed (no heads, GQA/MHA, or attention variant named)" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "no attention description at all" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: standard Transformer stack with residual connections implied; the loop is a second pass through the same sequential stack, not attention over prior layer outputs" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "positional encoding not disclosed" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "context length is extended by training at the deployed length — a three-stage SFT curriculum that progressively extends maximum training context from 64K to 128K and 256K tokens (§3.2.1); no positional-encoding extension method described" }, "factor": { "value": null, "provenance": "n/a", "ref": "§3.2.1", "note": "no rescaling factor applies; no PE extension method described" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§3.2.1 ('progressively extends the maximum training context from 64K to 128K and 256K tokens'); Appendix B.1 ('Context Window: 256k tokens')" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "placement": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "not disclosed" } }, "activation": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "no architecture disclosure to which the absence rule could apply" }, "softcapping": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "other": { "value": null, "provenance": "unknown", "note": "no block-level stability mechanisms disclosed" } } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§3.2.1 ('a three-stage curriculum that progressively extends the maximum training context from 64K to 128K and 256K tokens')", "note": "256K is reached in the SFT curriculum; pre-training context length is not separately disclosed" }, "deployed_length": { "value": 262144, "provenance": "stated", "ref": "Appendix B.1 ('Context Window: 256k tokens')" }, "extension_method": { "value": "three-stage SFT curriculum extending the maximum training context 64K → 128K → 256K (training at the deployed length); no positional-encoding extension method described", "provenance": "stated", "ref": "§3.2.1" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer not described" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 28, "provenance": "stated", "ref": "§2.2 ('Our pre-training corpus comprises 28T tokens')" }, "data_composition": { "value": "corpus exceeding Nanbeige 4.1 in scale and quality; increased sampling weights for mathematics, code, and synthetic QA data (found particularly beneficial for compact models); a small proportion of agentic trajectory data mixed into pre-training ('an initial step toward agentic pre-training')", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "single-stage from-scratch pre-training on 28T tokens (no staging disclosed); post-pre-training, the SFT recipe runs a three-stage curriculum (64K → 128K → 256K context) that shifts the supervised target-token mixture from STEM-centered reasoning (82.7% at 64K) through a transition stage (47.8% STEM / 29.5% agentic at 128K) to agentic-dominated (68.9% agentic / 22.4% STEM / 8.7% general at 256K)", "provenance": "stated", "ref": "§2.1, §3.2.1" }, "optimizer": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed (future-work section only mentions avoiding 'minimal additional FLOPs')" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.2.1 ('Starting from the pretrained checkpoint, we conduct supervised fine-tuning with a three-stage curriculum'); turn-level loss masking via execution feedback" }, "preference_optimization": { "value": "RLHF", "provenance": "stated", "ref": "§3.2.2 ('we first apply a two-stage RLHF procedure covering both think and non-think responses'; point-wise reward model over response quality)" }, "reasoning_training": { "value": "Think/Non-Think hybrid modes with two-stage RLHF; reasoning RL with a problem-dependent length-control objective (offline median-length budgets per problem, difficulty-aware penalty alternating constrained and free-expansion phases); agentic RL with action-centric rubrics providing turn-level process rewards plus outcome rewards, restricted to easy tasks (short trajectories, high pass@8) at this scale", "provenance": "stated", "ref": "§3.2.2, §3.2.3, §3.2.4" }, "distillation": { "value": null, "provenance": "unknown", "note": "no distillation mentioned; from-scratch pre-training followed by SFT/RLHF/RL" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes only text-based reasoning, code, tool interaction, and office-document work; no non-text modality appears" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§1", "note": "text-only; no non-text modality" } }, "efficiency": { "kv_cache": { "value": "full (non-sharing) two-pass loop: each pass through the shared layer stack keeps its own KV cache, so inference KV memory is roughly doubled versus a single-pass stack; the report explicitly rejected KV-cache sharing across loop passes because it halves the cache but reduces performance", "provenance": "derived", "note": "from the loop design in §2.1 ('we investigate a variant that shares the KV cache across loop passes... We therefore retain the full loop'); underlying attention mechanism itself undisclosed" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "no quantization format described" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding or multi-token prediction mentioned" }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "no serving optimisations disclosed (sglang appears only as the evaluation service, Appendix B.3)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "GDPval Rubrics", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 74.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AgentIF-Oneday", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 67.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "OfficeQA-Pro", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 21.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "PinchBench-V2", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 74.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ClawGym", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 65.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Claw-Eval Pass^3", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 52.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MCP-Atlas", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 57.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Bench Verified", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 63.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Bench Pro", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 46.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Terminal-Bench 2.0", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 44.1, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (w/o Search)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 17.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SciCode", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 35.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA Diamond", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 87.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HMMT-Feb-2026", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 82.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "IMO-Answer-Bench", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 67.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LiveCodeBench-V6", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 72.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 58.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "IF-Bench", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 54.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Recruit-Bench", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 63.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "First open-source model at this scale (3B non-embedding) combining code-agent, office-agent, and complex tool-use capabilities while retaining strong general reasoning", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } }, { "text": { "value": "Looped Transformer trained from scratch (two-pass loop reusing the layer stack), shown to outperform upcycling-based loop conversion; loop depth and KV-cache-sharing configurations studied", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Hybrid environment pool (real + synthesized) with execution-grounded trajectory synthesis across heterogeneous scaffolds (Claude Code, OpenHands, SWE-agent, Codex-based drivers) to induce scaffold-invariant strategies", "provenance": "stated", "ref": "§3.1" }, "ref": { "value": "§3.1", "provenance": "stated", "ref": "§3.1" } }, { "text": { "value": "Two-stage RLHF over Think and Non-Think responses, with findings of cross-task and cross-mode generalization of behavioral regularization (repetition/format/termination control)", "provenance": "stated", "ref": "§3.2.2" }, "ref": { "value": "§3.2.2", "provenance": "stated", "ref": "§3.2.2" } }, { "text": { "value": "Reasoning RL with a problem-dependent length-control objective (offline median-length budgets + difficulty-aware penalty) balancing accuracy and reasoning efficiency", "provenance": "stated", "ref": "§3.2.3" }, "ref": { "value": "§3.2.3", "provenance": "stated", "ref": "§3.2.3" } }, { "text": { "value": "Agentic RL with action-centric rubrics as turn-level process rewards (tool-call accuracy, per-turn information gain), stabilizing long-horizon training and cutting the single-turn action error rate by ~20%", "provenance": "stated", "ref": "§3.2.4" }, "ref": { "value": "§3.2.4", "provenance": "stated", "ref": "§3.2.4" } } ], "assessment": { "value": "The block-level differentiator is the two-pass Looped Transformer: no other atlas record reuses its own layer stack for a second pass to gain effective depth without parameters (the report cites the Loopy line of work, not in the atlas). Everything else — SFT curriculum, two-stage RLHF on Think/Non-Think, length-controlled reasoning RL, rubric-based agentic RL — is recipe work similar in spirit to the mini-activation agentic post-training of minimax-m2 but at 3B scale and with the length-control twist. The report is honest about its scope: it discloses almost no architecture hyperparameters, so the loop's exact depth/width and attention configuration cannot be compared against atlas peers. The claimed 'first at this scale' positioning (code + office + tool-use agents with retained reasoning) is plausible but not independently verifiable from the report.", "provenance": "inferred", "note": "comparison against atlas entries (nanbeige-4-1-3b, minimax-m2, qwen-qwen3-dense) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "Looped Transformer with a standard dense stack; no MoE described (§2.1)" }, "sequence_mixing": { "value": null, "provenance": "unknown", "note": "the report discloses no attention mechanism whatsoever; no efficient-attention or SSM claim appears, but full-attention cannot be verified from this report, so the tag is unknown rather than guessed (same treatment as sibling nanbeige-4-1-3b)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred text-only)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; training data not released" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "3B (non-embedding) lies in 1-10B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a 'compact general agentic model' maximizing general-purpose agentic capabilities while retaining strong reasoning (Abstract, §1) — a small generalist intent, same framing as sibling nanbeige-4-1-3b. Secondary intent: local/on-device personal-assistant deployment via OpenClaw (§3.3.2), noted in prose." } }, "prose": { "design_overview": { "value": "Nanbeige4.2-3B is a 3B non-embedding-parameter (4B total) dense model whose one structural idea is depth on a budget: a Looped Transformer that passes hidden states through the same Transformer stack twice, roughly doubling effective computational depth without adding parameters. The report is explicit that the loop is trained from scratch (upcycling a standard transformer into a loop tested worse), that two passes is the sweet spot (retaining about 75% of a standard transformer's token efficiency for a significant capacity gain), and that KV-cache sharing across passes was rejected: it halves inference KV memory but consistently underperforms the full loop, so the shipped model keeps separate caches per pass. Beyond that single architectural decision, the report discloses almost no block details — no layer count, hidden width, heads, vocabulary, normalization, activation, or positional encoding — and concentrates instead on the post-training recipe that makes a 3B model agentic: a three-stage SFT curriculum that grows context 64K → 128K → 256K while shifting the target-token mixture from 82.7% STEM reasoning to 68.9% agentic data; turn-level loss masking so the model learns recovery from failed turns; two-stage RLHF over Think and Non-Think responses with a point-wise reward model; reasoning RL with problem-dependent length budgets and a difficulty-aware penalty; and agentic RL whose action-centric rubrics feed turn-level process rewards alongside outcome rewards, cutting the action error rate by about 20%. The result is positioned as a compact general agent, deployable locally (OpenClaw), that beats Qwen3.5-9B and Gemma4-12B on the reported agentic suites.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Looped Transformer — reusing the shared layer stack for an additional pass (two-pass loop)", "provenance": "stated", "ref": "§2.1 (cites [2], the Loopy line of work)" }, "origin": { "value": "Looped Transformers (Bae et al.; not in atlas)", "provenance": "stated", "ref": "§2.1" } } ], "influence_out": [], "notable_omissions": { "value": [ "No architecture hyperparameters disclosed (layers, hidden width, heads, vocab, FFN dim, normalization, activation, positional encoding) despite 'we tune the depth of the transformer stack and its hidden width'", "No optimizer, learning-rate schedule, batch size, precision, parallelism, hardware, or compute disclosure", "Tokenizer not described", "No license named", "No release date printed (undated preprint)", "Pre-training context length not separately disclosed (only the SFT curriculum reaching 256K)", "The 4.1 → 4.2 delta is not quantified beyond loop architecture, data scale, and recipe changes", "RL algorithm details (which policy-gradient method underlies the reasoning/agentic RL stages) not named", "No embedding-tying or KV-cache-size statements for the underlying attention" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "nanbeige-4-2-3b.json" }, { "slug": { "value": "nvidia-nemotron-3-nano-30b-a3b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/nvidia-nemotron-3-nano-30b-a3b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Nemotron 3 Nano", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "NVIDIA", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2025-12-23", "provenance": "stated", "ref": "Title page date line (2025-12-23)" }, "report_url": { "value": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named anywhere in the report (only 'license-compliant' data filtering, §3.1.3)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We release both our pretrained Nemotron 3 Nano 30B-A3B Base and post-trained Nemotron 3 Nano 30B-A3B checkpoints on Hugging Face'" } }, "scale": { "total_params": { "value": 31.6, "provenance": "stated", "ref": "§2.1 ('31.6B total parameters'); Abstract" }, "active_params_per_token": { "value": 3.2, "provenance": "stated", "ref": "§2.1 ('3.2B are active (3.6B including embeddings) per forward pass'); Abstract ('activating less than half of the parameters per forward pass')" }, "layers": { "value": 52, "provenance": "stated", "ref": "Table 1 (Num Layers)" }, "hidden_dim": { "value": 2688, "provenance": "stated", "ref": "Table 1 (Model Dimension)" }, "ffn_inner_dim": { "value": 1856, "provenance": "stated", "ref": "Table 1 (Expert Dimension)" }, "ffn_ratio": { "value": 0.69, "provenance": "derived", "note": "1856 / 2688 = 0.69 (per-expert inner dim over hidden dim; all FFNs are MoE experts)" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 1 (Q-heads)" }, "attention_heads_kv": { "value": 2, "provenance": "stated", "ref": "Table 1 (KV-heads); §1 (GQA)" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 1 (Head Dimension)" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "report never discloses the vocabulary size or tokenizer" }, "embedding_tied": { "value": false, "provenance": "stated", "ref": "§2.1 ('un-tie embedding and projection weights')" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "report states standard FFN layers are replaced by sparse MoE layers throughout (§2.1: 'replacing the standard FFN layers with sparse Mixture-of-Experts (MoE) layers'); all 52 layers carry an MoE FFN" }, "moe": { "expert_count": { "value": 128, "provenance": "stated", "ref": "Table 1 (Total Routable Experts)" }, "experts_per_token": { "value": 6, "provenance": "stated", "ref": "Table 1 (Number of Activated Experts); §1 ('activates 6 out of 128 experts')" }, "shared_experts": { "value": 2, "provenance": "stated", "ref": "Table 1 (Number of Shared Experts)" }, "routing": { "value": "standard learnt MLP router with sigmoid gating; router weights frozen during RLVR to stabilize training", "provenance": "stated", "ref": "§2.1; §3.2.5" }, "load_balancing": { "value": "DeepSeek aux-loss-free load balancing (expert-bias based) with update rate 10−3, used in conjunction with the standard load balancing loss with coefficient 10−4; sequence-level MoE load balancing regularizer (coefficient 10−4) during SFT", "provenance": "stated", "ref": "§2.4; §3.1.6; §3.2.5" }, "expert_granularity": { "value": "granular MoE (DeepSeekMoE-style fine-grained experts) with shared experts: 128 routable experts of dimension 1856, 6 active per token, plus 2 shared experts", "provenance": "stated", "ref": "§2.1 ('granular MoE architecture along with shared experts (Dai et al., 2024)'); Table 1" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§1 ('combination of Mamba-2 (Dao & Gu, 2024) and Grouped-Query-Attention (GQA) (Ainslie et al., 2023)')" }, "attention_layer_pattern": { "value": "6 of 52 layers are GQA self-attention layers, the remaining 46 are Mamba-2; Figure 2 depicts a periodic interleaving of Mamba-2+MoE and Mamba-2+Attention+MoE blocks (repeat counts x5/x3/x1/x4 per the figure) — the extracted figure text does not fully resolve the exact interleaving, but §4.2 states 6 self-attention layers and that the 6 Mamba layers feeding them are kept in BF16", "provenance": "stated", "ref": "§4.2 ('self-attention layers (6 out of 52 layers for Nemotron 3 Nano)'); Figure 2" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: architecture described in full (§2.1); no learned attention over prior layer outputs described" }, "position_encoding": { "method": { "value": "NoPE", "provenance": "stated", "ref": "§2.1 ('We do not use any positional embeddings')" }, "rope_base": { "value": null, "provenance": "n/a", "ref": "§2.1 (no positional embeddings of any kind; no RoPE anywhere)" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "§2.1 (no RoPE)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "No positional-encoding modification applies (NoPE). Long-context ability comes from the LC-Phase: continuous pretraining (CPT) on a mixture of 512k and 4k sequences (§2.5); the model supports context lengths up to 1M tokens.", "ref": "§2.5" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.5 (no rescaling/interpolation factor; no RoPE)" }, "target_length": { "value": 1000000, "provenance": "stated", "ref": "§1, §5 ('supports context lengths up to 1M tokens'); RULER-100 @ 1M evaluated (Table 3)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.1 ('We use RMSNorm for normalization')" }, "placement": { "value": null, "provenance": "unknown", "note": "report does not describe pre/post norm placement" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "absence rule: no QK-normalisation described anywhere in the report" } }, "activation": { "value": "squared ReLU (on the MoE layers)", "provenance": "stated", "ref": "§2.1 ('For the MoE layers, we use squared ReLU activation')" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described" }, "other": [ { "value": "MoE router weights frozen during RLVR to stabilize training", "provenance": "stated", "ref": "§3.2.5 ('To further stabilize training we also freeze the MoE router weights')" } ] } }, "context": { "trained_length": { "value": 524288, "provenance": "stated", "ref": "§2.5 (LC-Phase CPT on batches of 524,288 (512k) token sequences, mixed with 4k sequences; base pretraining at 8192)" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "§1 ('supports context lengths up to 1M tokens'); RULER-100 @ 1M (Table 3)" }, "extension_method": { "value": "LC-Phase continuous pretraining (CPT): 512k and 4k sequence mixture, 121B tokens, with 20% long-context document QA and 1% synthetic retrieval-focused data (up to 256k); no positional-encoding modification needed (NoPE)", "provenance": "stated", "ref": "§2.5" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "report never describes the tokenizer" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 25, "provenance": "stated", "ref": "Abstract ('pretrained on 25 trillion text tokens'); §2.4; note: 23.5T phase 1 + 1.5T phase 2 + 121B LC-Phase (§1, §2.5)" }, "data_composition": { "value": "15 data categories: web crawl subdivided into five quality groups (crawl-medium, crawl-medium-high, syn-crawl-medium-high, crawl-high, syn-crawl-high) plus math, Wikipedia, code, nemotron-cc-code, academic text, Crawl++ (OpenWebText, BigScience, Reddit), multilingual (19 languages), and synthetic SFT-style datasets (general-sft, stem-sft, code-sft); new released datasets: Nemotron-CC-v2.1 (~2.5T new English tokens: 3 fresh Common Crawl snapshots, 5-prompt rephrasing of Medium-High data from 110 snapshots, translation to English from 9 languages), Nemotron-CC-Code-v1 (427.92B-token code corpus via Lynx + Phi-4 cleaning), Nemotron-Pretraining-Code-v2 (GitHub refresh + Qwen3-32B synthetic dialogues, SGCR/SCOR rephrasing, Python-to-C++ transpilation), Nemotron-Pretraining-Specialized-v1 (synthetic Wikipedia, math textbooks, scientific coding, InfiniByte cross-domain code, RQA 4.3M demonstrations / ~31.7B tokens, DQA ~8B tokens)", "provenance": "stated", "ref": "§2.2, §2.3" }, "curriculum": { "value": "two-phase curriculum: Phase 1 promotes diversity, Phase 2 primarily high-quality datasets (e.g., Wikipedia), switch at the 94% point of training; LC-Phase CPT at the end (512k/4k mixture); RLVR uses a difficulty curriculum with Gaussian pass-rate sampling that shifts from easy to hard tasks across training steps", "provenance": "stated", "ref": "§2.3, §2.5, §3.2.2" }, "optimizer": { "value": "AdamW with weight decay 0.1, β1 = 0.9, β2 = 0.95", "provenance": "stated", "ref": "§2.4" }, "lr_schedule": { "value": "Warmup-Stable-Decay: warmed up over 8.4B tokens to a maximum of 10−3, maintained for 80% of training (20T tokens), decayed to a minimum of 10−5 over the final 5T tokens; LC-Phase used a constant LR of 10−5", "provenance": "stated", "ref": "§2.4, §2.5" }, "batch_schedule": { "value": "pretraining: sequence length 8192, batch size 3072 (~25M tokens per batch); LC-Phase: global batch size 48; SFT: batch size 64 with sequence packing to 256K; RLVR: batch size 2048 (128 prompts/step, 16 generations/prompt)", "provenance": "stated", "ref": "§2.4, §2.5, §3.1.6, §3.2.5" }, "precision": { "value": "BF16", "provenance": "stated", "ref": "§4 ('After post-training the model in BF16, we applied Post-Training Quantization (PTQ) using ModelOpt and Megatron-LM to quantize the model to FP8')" }, "parallelism": { "value": "LC-Phase: 8-way context parallelism, 8-way tensor parallelism, 8-way expert parallelism, 4-way pipeline parallelism; main pretraining parallelism not disclosed", "provenance": "stated", "ref": "§2.5" }, "hardware": { "value": "H100 GPUs (disclosed only for the LC-Phase; main pretraining hardware not stated)", "provenance": "stated", "ref": "§2.5 ('to train on H100 GPUs'); throughput measured on a single H200 GPU (§1)" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (SFT over 18M samples; 13000 steps, batch size 64, sequence packing to 256K, LR 5·10−5 with 800 warmup steps; chat/agentic/reasoning traces; reasoning on/off control by stripping traces from 10% of samples, budget control by truncating 3%)" }, "preference_optimization": { "value": "RLHF", "provenance": "stated", "ref": "§3.3 (RLHF with a generative reward model (GenRM) built by GRPO-training Qwen3-235B-A22B-Thinking-2507; Group Relative Length Control with length-normalized reward adjustment and quality-gated conciseness bonus; verbosity reduced 30% without accuracy loss)" }, "reasoning_training": { "value": "multi-environment RLVR from verifiable rewards: all environments trained simultaneously (competition math DAPO/SkyWorks, competition coding, STEM MCQ QA, structured outputs/JSON schema, instruction following, long context, agentic tool use incl. Workplace Assistant and multi-turn conversational agent); synchronous GRPO with masked importance sampling, 128 prompts/step, 16 generations/prompt, batch 2048, max generation length 49K, overlong filtering, difficulty curriculum; two RLVR stages (after SFT and after RLHF); reasoning on/off and token-budget control via the chat template", "provenance": "stated", "ref": "§3.2" }, "distillation": { "value": "none", "provenance": "inferred", "note": "no weight-level distillation from another model described; teacher models (GPT-OSS-120B, DeepSeek-R1-0528, Qwen3-235B-A22B-Thinking-2507, Qwen3-Coder-480B, Kimi-K2-Instruct-0905, Goedel-Prover-V2-32B, etc.) are used only to generate SFT/RL data and trajectories (§3.1.2) — teacher-generated data, noted in prose per schema" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes no vision or other non-text modality; all training data and evaluation benchmarks are text-based" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": "GQA with 2 KV heads (KV cache ∝ 2 heads); KV cache quantized to FP8 via PTQ — FP8 KV cache significantly improves throughput by enabling larger batch sizes", "provenance": "stated", "ref": "§4.2 ('The model weights, activations, and KV cache are quantized to FP8'), §4.3" }, "quantization": { "value": "FP8 post-training quantization (PTQ) via ModelOpt and Megatron-LM with selective BF16 retention: the 6 self-attention layers, the 6 Mamba layers feeding them, and all Mamba Conv1D layers kept in BF16; weights, activations, and KV cache FP8; ~99% median accuracy recovery", "provenance": "stated", "ref": "§4" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "absence rule: report is silent on speculative decoding or multi-token prediction" }, "serving": { "value": "inference throughput measured with vLLM and TRT-LLM (best of the two per model) on a single H200 GPU with FP8 weights and activations; no dedicated serving-optimization stack described", "provenance": "stated", "ref": "§1, Figure 1 caption" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 78.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME25 (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.06, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME25 (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 99.17, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 73.04, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 75.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LiveCodeBench (v6 2024-08↔2025-05)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 68.25, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SciCode (subtask)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 33.28, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 10.57, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 15.48, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MiniF2F pass@1", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 50.03, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MiniF2F pass@32", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 79.92, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Terminal Bench (hard subset)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 8.51, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Bench (OpenHands)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 38.76, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "TauBench V2 Airline", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 48.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "TauBench V2 Retail", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 56.91, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "TauBench V2 Telecom", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 42.21, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "TauBench V2 Average", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 49.04, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BFCL v4", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 53.76, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "IFBench (prompt)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 71.51, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Scale AI Multi Challenge", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 38.45, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Arena-Hard-V2 (Average)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 67.65, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 35.85, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "RULER-100 @ 256k", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 92.92, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "RULER-100 @ 512K", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 91.25, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "RULER-100 @ 1M", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 86.34, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU-ProX (avg over langs)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 59.5, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "WMT24++ (en→xx)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 86.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Open, efficient MoE hybrid Mamba-Transformer for agentic reasoning: 31.6B total / 3.2B active parameters (3.6B including embeddings), better or on-par accuracy vs GPT-OSS-20B and Qwen3-30B-A3B-Thinking-2507 with up to 3.3× higher inference throughput (8K input / 16K output on a single H200)", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } }, { "text": { "value": "Granular MoE with shared experts: 128 routable experts (6 active per token) with squared ReLU and a learnt MLP sigmoid router, achieving 'better accuracy at a fraction of the active parameter count'", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "25T-token pretraining with new open datasets: Nemotron-CC-v2.1 (2.5T new English tokens incl. translation from 9 languages), Nemotron-CC-Code-v1 (427.92B code tokens), Nemotron-Pretraining-Code-v2, and Nemotron-Pretraining-Specialized-v1 (incl. InfiniByte cross-domain problem synthesis and 4.3M RQA STEM reasoning demonstrations)", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "First Nemotron effort to scale RL in post-training: unified multi-environment RLVR training on all environments simultaneously, followed by RLHF with a GRPO-trained generative reward model and a novel Group Relative Length Control mechanism that reduced verbosity 30% without sacrificing accuracy", "provenance": "stated", "ref": "§3" }, "ref": { "value": "§3", "provenance": "stated", "ref": "§3" } }, { "text": { "value": "Selective FP8 PTQ keeping the 6 self-attention layers and their 6 feeding Mamba layers in BF16: ~99% median accuracy recovery with FP8 KV cache and significantly higher throughput", "provenance": "stated", "ref": "§4" }, "ref": { "value": "§4", "provenance": "stated", "ref": "§4" } }, { "text": { "value": "Open release: base and post-trained checkpoints on HuggingFace, plus training recipe, code, and 'most of the data' used to train the model", "provenance": "stated", "ref": "Abstract, §5" }, "ref": { "value": "Abstract, §5", "provenance": "stated", "ref": "Abstract, §5" } } ], "assessment": { "value": "Architecturally an incremental combination of established mechanisms: the Mamba-2 + GQA hybrid follows the Jamba / Nemotron-H / Nemotron 2 Nano lineage, the granular MoE with shared experts follows DeepSeekMoE, and the aux-loss-free bias-based load balancing follows DeepSeek-V3. The genuinely distinctive contributions are in post-training and efficiency: unified multi-environment RLVR (all environments simultaneously), RLHF with a generative reward model plus Group Relative Length Control (a new, simple reward shaping that cut verbosity 30% with no accuracy loss), and a selective FP8 PTQ scheme (6 attention + 6 feeding Mamba layers kept BF16). Squared ReLU MoE at 6/128 routing is a notable but minor variant. Nothing here is a new mixing mechanism; the report's headline is the accuracy/throughput trade-off for an open 30B-class agentic model.", "provenance": "inferred", "note": "comparison against atlas entries (jamba, deepseek-v3, mixtral, llama-3.1, phi-3, moonshot-kimi-k3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all 52 layers use MoE FFNs ('replacing the standard FFN layers with sparse Mixture-of-Experts (MoE) layers', §2.1)" }, "sequence_mixing": { "value": "SSM", "provenance": "derived", "note": "46 of 52 layers are Mamba-2, 6 are GQA attention (§4.2); Mamba share 46/52 = 88.5% ≥ 80% threshold" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights-open-data", "provenance": "derived", "note": "open_weights stated true; report also states it provides 'the recipe, code, and most of the data we used to train the model' and releases 'the vast majority of the new data on HuggingFace' (§1, §2.2, §5)" }, "scale_class": { "value": "large", "provenance": "derived", "note": "31.6B total params in [10B, 100B)" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "title frames it as 'an Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning'; abstract emphasizes agentic, reasoning, and chat abilities with reasoning on/off and budget control, and throughput efficiency. Secondary intents: long-context (1M) and efficient on-device-class serving, noted in prose." } }, "prose": { "design_overview": { "value": "Nemotron 3 Nano 30B-A3B is an open, efficient sparse MoE hybrid Mamba-Transformer: 52 layers, 31.6B total parameters with only 3.2B active per forward pass (3.6B including embeddings). The backbone follows the Nemotron-H / Nemotron 2 Nano lineage — a Jamba-style interleaving of Mamba-2 state-space layers and Grouped-Query-Attention layers, 6 of 52 layers being GQA (32 query heads, 2 KV heads, head dimension 128) and the remaining 46 being Mamba-2 (state dimension 128, 8 groups, 64 heads) — with every standard FFN replaced by a granular MoE: 128 routable experts of dimension 1856, 6 activated per token plus 2 shared experts, a learnt MLP sigmoid router, squared ReLU activation, and DeepSeek's aux-loss-free bias-based load balancing combined with a small auxiliary loss. The model is fully NoPE: no positional embeddings, no dropout, no linear-layer bias, RMSNorm, untied embeddings. Context reaches 1M tokens through a final LC-Phase of continuous pretraining on a 512k/4k sequence mixture, with no positional-encoding extension required. Post-training runs SFT over chat, agentic, and reasoning traces with reasoning on/off and token-budget control, a unified multi-environment RLVR stage (synchronous GRPO with masked importance sampling and frozen router weights), and RLHF with a GRPO-trained generative reward model plus Group Relative Length Control that cut verbosity 30% without accuracy loss. Deployment ships BF16 and a selective FP8 PTQ checkpoint that keeps the 6 attention layers and their feeding Mamba layers in BF16, recovering ~99% median accuracy.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Hybrid Mamba-Transformer architecture (Mamba-2 + GQA interleaving)", "provenance": "stated", "ref": "§1 (cites Jamba (Lieber et al., 2024), Mamba-2 (Dao & Gu, 2024), GQA (Ainslie et al., 2023)); §2.1 (builds on Nemotron-H and Nemotron 2 Nano)" }, "origin": { "value": "Jamba / Nemotron-H / Nemotron 2 Nano", "provenance": "stated", "ref": "§1, §2.1" } }, { "mechanism": { "value": "Granular MoE with shared experts (fine-grained expert specialization)", "provenance": "stated", "ref": "§2.1 (cites Dai et al., 2024 — DeepSeekMoE)" }, "origin": { "value": "DeepSeekMoE", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Aux-loss-free bias-based load balancing (expert-bias update rate 10−3)", "provenance": "stated", "ref": "§2.4 (cites Wang et al., 2024; DeepSeek-AI, 2025b — DeepSeek-V3)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.4" } }, { "mechanism": { "value": "Sparse Mixture-of-Experts layers", "provenance": "stated", "ref": "§1, §2.1 (cites Shazeer et al., 2017)" }, "origin": { "value": "Shazeer et al. (MoE, not in atlas)", "provenance": "stated", "ref": "§1" } }, { "mechanism": { "value": "Warmup-Stable-Decay learning rate schedule", "provenance": "stated", "ref": "§2.4 (cites Hu et al., 2024)" }, "origin": { "value": "MiniCPM (Hu et al., 2024)", "provenance": "stated", "ref": "§2.4" } }, { "mechanism": { "value": "Generative reward model (GenRM) trained with RL (GRPO) for RLHF", "provenance": "stated", "ref": "§3.3.1 (builds on Wang et al., 2025b; Liu et al., 2025b)" }, "origin": { "value": "HelpSteer3 / RLBFF (Wang et al., 2025b, not in atlas)", "provenance": "stated", "ref": "§3.3.1" } } ], "influence_out": [ { "model_slug": { "value": "nvidia-nemotron-3-super-120b-a12b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Hybrid Mamba-2 + GQA backbone with MoE FFNs", "provenance": "inferred", "note": "mechanism as stated in nvidia-nemotron-3-super-120b-a12b report's lineage_in" } } ], "notable_omissions": { "value": [ "No vocabulary size or tokenizer algorithm disclosed", "No license named for the released weights", "Main 25T-token pretraining hardware and parallelism not disclosed (only the LC-Phase: H100, 8-way context/tensor/expert parallelism, 4-way pipeline)", "No total training compute (FLOPs or GPU-hours) disclosed", "Normalization placement (pre/post) and QK-norm not described", "Exact attention-layer interleaving ambiguous: text states 6 of 52 layers are self-attention (§4.2), but Figure 2's repeat counts (x5/x3/x1/x4) do not obviously reconcile with that count in the extracted figure text", "No speculative decoding or multi-token prediction (unlike Nemotron 3 Super)", "Mamba SSM parameterization (discretization, decay initialization) not described despite Table 1 giving state dimension/groups/heads", "Base-model comparison to GPT-OSS-20B omitted because no GPT-OSS base model was released (report states this explicitly)", "Peak learning rate and batch size are disclosed, but no loss-curve or scaling-law data" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "nvidia-nemotron-3-nano-30b-a3b.json" }, { "slug": { "value": "nvidia-nemotron-3-super-120b-a12b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/nvidia-nemotron-3-super-120b-a12b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Nemotron 3 Super", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "NVIDIA", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-04-03", "provenance": "stated", "ref": "Title page date line (printed '2026-4-3')" }, "report_url": { "value": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named anywhere in the report" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('Nemotron 3 Super datasets, along with the base, post-trained, and quantized checkpoints, are open-sourced on HuggingFace'); §5" } }, "scale": { "total_params": { "value": 120.6, "provenance": "stated", "ref": "§2.1 ('We extend this foundation to 120.6B total parameters'); Abstract ('120 billion (active 12 billion) parameter')" }, "active_params_per_token": { "value": 12.7, "provenance": "stated", "ref": "§2.1 ('maintaining a constrained active budget of 12.7B parameters (12.1B excluding embeddings) per forward pass')" }, "layers": { "value": 88, "provenance": "stated", "ref": "Table 1 (Total Layers)" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 1 (Model Dimension)" }, "ffn_inner_dim": { "value": 2688, "provenance": "stated", "ref": "Table 1 (Expert Hidden Dimension — per routed expert; shared experts have intermediate size 5376)" }, "ffn_ratio": { "value": 0.66, "provenance": "derived", "note": "2688 / 4096 = 0.66 (per-expert inner dim over hidden dim; shared-expert ratio would be 5376/4096 = 1.31)" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 1 (Q-Heads); §2.1.3" }, "attention_heads_kv": { "value": 2, "provenance": "stated", "ref": "Table 1 (KV-Heads); §2.1.3 ('Grouped-Query Attention (GQA) with 32 query heads and 2 KV heads')" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 1 (Head Dimension); §2.1.3" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "report never discloses the vocabulary size or tokenizer" }, "embedding_tied": { "value": false, "provenance": "stated", "ref": "§2.1.3 ('maintain un-tied embedding and output weights')" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "all 88 layers pair Mamba-2/attention with a LatentMoE FFN ('the 88-layer stack follows a periodic interleaving pattern in which MoE layers are paired with Mamba-2 blocks', §2.1.3; 'sparse LatentMoE scaling', §2.1)" }, "moe": { "expert_count": { "value": 512, "provenance": "stated", "ref": "Table 1 (Total Experts per Layer); §2.4 ('MoE layers with 512 total experts')" }, "experts_per_token": { "value": 22, "provenance": "stated", "ref": "Table 1 (Top-k (Activated Experts) 22); §2.1.3 ('top-22 routing')" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "shared experts exist ('shared expert computation', §2.1.1; 'Shared Expert Intermediate Size 5376', Table 1) but their count is never disclosed" }, "routing": { "value": "sigmoid router score function complemented by expert biasing; the routing gate operates in the full hidden dimension (not the latent space)", "provenance": "stated", "ref": "§2.4; §2.1.1" }, "load_balancing": { "value": "auxiliary-loss-free load balancing strategy (expert-bias based, update rate 10−3) paired with a standard load balancing loss with coefficient 10−4", "provenance": "stated", "ref": "§2.4 (cites Wang et al., 2024; DeepSeek-AI, 2025c; Lepikhin et al., 2020)" }, "expert_granularity": { "value": "LatentMoE: each token is down-projected from hidden dim 4096 into a 1024-wide latent space (MoE Latent Size; d/ℓ = 4 compression) where all 512 routed experts (expert hidden dim 2688, top-22) operate; expert count and top-k are scaled up by d/ℓ relative to a standard MoE; router, shared experts, and non-expert layers remain in full hidden dimension", "provenance": "stated", "ref": "§2.1.1, Table 1" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2.1.3 ('The attention layers employ Grouped-Query Attention (GQA) with 32 query heads and 2 KV heads (head dimension 128)')" }, "attention_layer_pattern": { "value": "periodic hybrid: the predominantly Mamba-2 stack has a limited number of GQA self-attention layers inserted as global 'anchors' for full-token interaction; Figure 2 shows a 22-layer period of Mamba-2+LatentMoE and Mamba-2+Attention+LatentMoE blocks with counts x4/x3/x1/x4/x3/x3/x4 (7 attention-containing blocks per period), which repeats over the 88-layer stack (88 = 4 × 22, ≈ 28 attention layers); the extracted figure text is partially ambiguous about the trailing count assignments", "provenance": "stated", "ref": "§2.1.3, Figure 2, Table 1" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: architecture described in full (§2.1); no learned attention over prior layer outputs described" }, "position_encoding": { "method": { "value": "NoPE", "provenance": "stated", "ref": "§2.1.3 ('we omit positional embeddings')" }, "rope_base": { "value": null, "provenance": "n/a", "ref": "§2.1.3 (no positional embeddings of any kind; no RoPE anywhere)" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "§2.1.3 (no RoPE)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "No positional-encoding modification applies (NoPE). Long-context ability comes from the LC-Phase continuous pretraining: 1,048,576-token sequences (34B tokens), then alternating 1m/4k sequences (17B tokens).", "ref": "§2.6" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.6 (no rescaling/interpolation factor; no RoPE)" }, "target_length": { "value": 1000000, "provenance": "stated", "ref": "§2.1.3 ('This configuration supports context lengths of up to 1M tokens'); RULER 1M evaluated (Table 4, Table 5)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.1.3 ('use RMSNorm for normalization')" }, "placement": { "value": null, "provenance": "unknown", "note": "report does not describe pre/post norm placement" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "absence rule: no QK-normalisation described anywhere in the report" } }, "activation": { "value": null, "provenance": "unknown", "note": "no activation function (ReLU/SiLU/GELU etc.) is named anywhere in the report, including for the LatentMoE experts" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described" }, "other": [ { "value": "Final 15% of the network kept in BF16 during NVFP4 pretraining to 'promote training stability at scale'", "provenance": "stated", "ref": "Table 3" }, { "value": "Mamba output projection kept in MXFP8 (not NVFP4) to mitigate the high incidence of underflows observed when quantizing this layer to NVFP4 at smaller scales", "provenance": "stated", "ref": "Table 3" } ] } }, "context": { "trained_length": { "value": 1048576, "provenance": "stated", "ref": "§2.6 (LC-Phase CPT on 1,048,576 (1m) context length for 34B tokens, then a stage alternating 1m and 4k sequences for 17B tokens)" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "§2.1.3 ('supports context lengths of up to 1M tokens'); RULER 1M (Table 4, Table 5)" }, "extension_method": { "value": "LC-Phase continuous pretraining (CPT): stage 1 at 1M sequence length (34B tokens, 20% long-context document QA + 80% downscaled Phase 2 data), stage 2 alternating 1M and 4k sequences (17B tokens) to mitigate impact on math benchmarks; no positional-encoding modification needed (NoPE)", "provenance": "stated", "ref": "§2.6" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "report never describes the tokenizer" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 25, "provenance": "stated", "ref": "Abstract ('pre-trained Nemotron 3 Super on 25 trillion tokens'); §1 (20T phase 1 + 5T phase 2)" }, "data_composition": { "value": "16 high-level categories (Nemotron 3 Nano mixture): web crawl in five quality groups (crawl-medium, crawl-medium-high, crawl-high, syn-crawl-medium-high, syn-crawl-high), math, Wikipedia, code, Nemotron-CC-Code, academic text, Crawl++ (OpenWebText, BigScience, Reddit), multilingual, finepdfs, and synthetic SFT-style data (general-sft, stem-sft, code-sft) including reasoning-focused datasets; new released datasets (Nemotron-Pretraining-Specialized-v1.1): Synthetic Code Concepts (15M Python problems from 91 concepts via GPT-OSS-20B/120B), Synthetic Unconditional Algorithmic (0.2B tokens, decontaminated against HumanEval/MBPP/CRUXEval/LiveCodeBench), Synthetic Economics, Synthetic Formal Logic, and Synthetic Multiple Choice (~3.5M MMLU-style MCQ samples, ~1.6B tokens, majority-voted via DeepSeek-V3)", "provenance": "stated", "ref": "§2.3" }, "curriculum": { "value": "two-phase curriculum (Feng et al., 2024): Phase 1 emphasizes diversity (80% of training, 20T tokens), Phase 2 shifts to predominantly high-quality sources (20%, 5T tokens), transition at 80% of total training tokens; LC-Phase CPT (1M then 1M/4k) at the end; checkpoint merging (weighted sliding-window averaging with minus-sqrt decay emulation) used for evaluation readouts, with the final base model being a 500B merge; RLVR uses a difficulty-based curriculum", "provenance": "stated", "ref": "§2.3.7, §2.5, §2.6, §3.2.1" }, "optimizer": { "value": "AdamW with weight decay 0.1 and momentum coefficients β1 = 0.9, β2 = 0.95", "provenance": "stated", "ref": "§2.4" }, "lr_schedule": { "value": "Warmup-Stable-Decay (WSD): warmed up over the initial 200B tokens to a peak of 4.5×10−4, sustained stable plateau, then minus-sqrt decay over the final 5T tokens annealing to a minimum of 4.5×10−6; LC-Phase used a constant LR of 4.5×10−6", "provenance": "stated", "ref": "§2.4, §2.6" }, "batch_schedule": { "value": "pretraining: sequence length 8,192 and batch size 3,072 sequences (~25.17M tokens per batch); LC-Phase: global batch size 16; SFT stage 1: batch size 64 with 256k sequence packing; SFT stage 2: batch size 32 with 512k packing; RLVR: batch size 4096 (256 prompts/step, 16 responses/prompt)", "provenance": "stated", "ref": "§2.4, §2.6, §3.1, §3.2.4" }, "precision": { "value": "NVFP4 pretraining (hybrid BF16/NVFP4): all linear layers trained in NVFP4 (E2M1 with 16-element micro-blocks, E4M3 block scales, FP32 global scale; 2D block scaling for weights, 1D for gradients/activations, random Hadamard transforms on wgrad inputs, stochastic rounding on gradients) except the final 15% of the network, latent projections, MTP layers, QKV & attention projections, and embeddings in BF16; Mamba output projection in MXFP8", "provenance": "stated", "ref": "§2.2, Table 3" }, "parallelism": { "value": "LC-Phase: 64-way context parallelism, 2-way tensor parallelism, 64-way expert parallelism; RL post-training: large-scale asynchronous training on thousands of GPUs (up to 1k-GPU scale discussed) with in-flight weight updates; main pretraining parallelism not disclosed", "provenance": "stated", "ref": "§2.6, §3.2.4, §3.2.5" }, "hardware": { "value": "GB200 GPUs (disclosed only for the LC-Phase); B200 used for PTQ (single 8-GPU node) and throughput measurement; B300 for MTP serving experiments; main pretraining hardware not disclosed", "provenance": "stated", "ref": "§2.6, §4.2, Figure 1 caption, Figure 5 caption" }, "compute": { "value": null, "provenance": "unknown", "note": "no absolute FLOPs or GPU-hours disclosed; only relative: checkpoint merging saves ~4T tokens of compute, ~16% of the total pretraining FLOP budget (§2.5)" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (two-stage SFT over 7M samples / 80B tokens: Stage 1 token-level average loss with 256k packing, global batch 64, constant lr 1e−5 with 30k warmup samples; Stage 2 per-conversation normalized loss with 512k packing, global batch 32, lr 1e−5, long-context data up to 512K; MTP objective continued with 0.3-scaled auxiliary loss; low-effort reasoning mode added, 2% of SFT data; reasoning-off via 3% trace stripping; 350-step semi-on-policy budget-control stage truncating 12% of traces)" }, "preference_optimization": { "value": "RLHF", "provenance": "stated", "ref": "§3.2.3 (separate RLHF-only stage at the end of post-training, plus GenRM used throughout multi-environment RL; principle-following GenRM initialized from Qwen3-235B-A22B-Thinking-2507, trained on Helpsteer 3, commercially friendly lmarena-140k subsets, and recent human preference data)" }, "reasoning_training": { "value": "three-stage RL + MTP healing: (1) multi-environment RLVR over 21 environments / 37 datasets (math with/without Python tool, code, STEM, instruction following, safety incl. PAIR-based jailbreak attacks, long context, agentic tool use, Reasoning Gym) with asynchronous GRPO and masked importance sampling, 256 prompts/step, 16 responses/prompt, batch 4096, max generation length 49K rising to 64K; (2) SWE-RL: end-to-end RL for GitHub-issue solving with Apptainer containers, OpenHands agent loop, binary test reward, OpenCode/Codex harness classes; (3) RLHF; then MTP healing (MTP heads trained with frozen backbone on RLVR prompts); PivotRL (assistant-turn-level RL reusing offline SFT expert trajectories on pivots) applied to all agentic domains; three reasoning modes (reasoning-off, regular, low-effort) with inference-time budget control", "provenance": "stated", "ref": "§3.2" }, "distillation": { "value": "none", "provenance": "inferred", "note": "no weight-level distillation from another model described; SFT/RL data and trajectories are teacher-generated (GPT-OSS-120B, Qwen3-Coder-480B-A35B-Instruct, MiniMax M2.5, DeepSeek-V3.2, Kimi K2, GLM-4.7, DeepSeek-R1-0528, etc.) — teacher-generated data, noted in prose per schema" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes no vision or other non-text modality; all training data and evaluation benchmarks are text-based" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": "GQA with 2 KV heads (KV cache ∝ 2 heads); KV cache quantized to FP8 in both PTQ checkpoints; Mamba SSM (state) cache quantized to FP16 with stochastic rounding (Philox<5>) — naive FP16 casting caused up to 40% verbosity increase because recurrent quantization error accumulates over decoding steps", "provenance": "stated", "ref": "§4.1, §4.2, §4.3" }, "quantization": { "value": "two PTQ deployment checkpoints via Model-Optimizer: FP8 (W8A8) for Hopper (MoE GEMMs and Mamba linear layers FP8, KV cache FP8, SSM cache FP16, attention/embedding BF16, router FP32) and NVFP4 (W4A4) for Blackwell (MSE-calibrated per-block weight scales, dynamic per-block max-based activation scales, and Model-Optimizer AutoQuantize mixed-precision search with 4.75-bit effective budget — sparse-expert GEMMs all NVFP4, attention/Mamba projections FP8 or BF16, shared experts mixed); NVFP4 model achieves 99.8% median accuracy vs BF16; full mixed-precision PTQ completed in under 2 hours on a single B200 node (8 GPUs, 512 SFT samples at sequence length 4096)", "provenance": "stated", "ref": "§4" }, "speculative_dedup": { "value": "native speculative decoding via Multi-Token Prediction: 2 MTP layers with shared weights (unified head exposed to multiple offsets, applied recursively at inference for longer drafts); average acceptance length 3.45 on SPEED-Bench at draft length 7; draft depths 1–3 shift the throughput-latency Pareto frontier on Blackwell (TRT-LLM, TP=1, B300)", "provenance": "stated", "ref": "§2.1.2, Table 2, Figure 5" }, "serving": { "value": "throughput measured with vLLM and TRT-LLM on B200 GPUs (best of the two per model; GPT-OSS-120B run MXFP4/MXFP8/FP8-KV, Qwen3.5-122B BF16); MTP serving experiments with TRT-LLM TP=1 on B300; vLLM/TRT-LLM quantized MoE APIs require shared-format sparse-expert groups (handled by AutoQuantize constraints)", "provenance": "stated", "ref": "Figure 1 caption, §4.2, Figure 5 caption" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 83.73, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "AIME25 (no tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 90.21, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HMMT Feb25 (no tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 93.67, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HMMT Feb25 (with tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 94.73, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "GPQA (no tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 79.23, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "GPQA (with tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 82.7, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "LiveCodeBench (v5 2024-07↔2024-12)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 81.19, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SciCode (subtask)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 42.05, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HLE (no tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 18.26, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "HLE (with tools)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 22.82, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Terminal Bench (hard subset)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 25.78, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Terminal Bench Core 2.0", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 31.0, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SWE-Bench (OpenHands)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 60.47, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SWE-Bench (OpenCode)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 59.2, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SWE-Bench (Codex)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 53.73, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "SWE-Bench Multilingual (OpenHands)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 45.78, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "TauBench V2 Airline", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 56.25, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "TauBench V2 Retail", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 62.83, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "TauBench V2 Telecom", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 64.36, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "TauBench V2 Average", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 61.15, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BrowseComp with Search", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 31.28, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "BIRD Bench", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 41.8, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "IFBench (prompt)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 72.56, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Scale AI Multi-Challenge", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 55.23, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Arena-Hard-V2", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 73.88, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 58.31, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "RULER 256k", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 96.83, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "RULER 512k", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 95.22, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "RULER 1M", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 91.64, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MMLU-ProX (avg over langs)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 79.36, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "WMT24++ (en→xx)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 86.67, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } } ] }, "contributions": { "claimed": [ { "text": { "value": "First model in the Nemotron 3 family to be pre-trained in NVFP4: stable and accurate low-precision pretraining over 25T tokens (NVFP4 GEMM kernels from Transformer Engine, final 15% of network kept BF16)", "provenance": "stated", "ref": "Abstract, §2.2" }, "ref": { "value": "Abstract, §2.2", "provenance": "stated", "ref": "Abstract, §2.2" } }, { "text": { "value": "LatentMoE — a new Mixture-of-Experts architecture optimizing accuracy per FLOP and per parameter: tokens projected from hidden dim d into a smaller latent dimension ℓ for routing and expert computation, with expert count and top-k scaled up by d/ℓ for higher accuracy at approximately constant inference cost", "provenance": "stated", "ref": "§2.1.1" }, "ref": { "value": "§2.1.1", "provenance": "stated", "ref": "§2.1.1" } }, { "text": { "value": "MTP layers with shared weights for native speculative decoding: a unified prediction head exposed to multiple offsets that can be applied recursively at inference for longer drafts without an external draft model (highest average acceptance length 3.45 on SPEED-Bench at draft length 7)", "provenance": "stated", "ref": "§2.1.2, Table 2" }, "ref": { "value": "§2.1.2, Table 2", "provenance": "stated", "ref": "§2.1.2, Table 2" } }, { "text": { "value": "Hybrid interleaved architecture with strategic GQA attention anchors over a predominantly Mamba-2 backbone: 88 layers, 120.6B total / 12.7B active parameters, 1M-token context, combining linear-time Mamba sequence modeling with full-token interaction", "provenance": "stated", "ref": "§2.1.3" }, "ref": { "value": "§2.1.3", "provenance": "stated", "ref": "§2.1.3" } }, { "text": { "value": "Checkpoint merging (offline minus-sqrt decay emulation) as a compute-saving evaluation technique during the stable LR phase: saves ~4T tokens of compute (~16% of the total pretraining FLOP budget); the final base model selected for alignment was itself a 500B merge", "provenance": "stated", "ref": "§2.5" }, "ref": { "value": "§2.5", "provenance": "stated", "ref": "§2.5" } }, { "text": { "value": "Scaled agentic post-training: 21 RL environments / 37 datasets, large-scale asynchronous RL on thousands of GPUs with in-flight weight updates and masked importance sampling, a dedicated end-to-end SWE-RL stage, and PivotRL for efficient long-horizon agentic RL", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } }, { "text": { "value": "FP8 and NVFP4 deployment checkpoints: NVFP4 PTQ combining MSE-calibrated weight scales, dynamic max-based activation scales, and AutoQuantize mixed-precision search (99.8% median accuracy vs BF16), plus an FP16-with-stochastic-rounding SSM cache recipe that eliminates recurrent quantization drift", "provenance": "stated", "ref": "§4" }, "ref": { "value": "§4", "provenance": "stated", "ref": "§4" } }, { "text": { "value": "Up to 2.2× and 7.5× higher inference throughput than GPT-OSS-120B and Qwen3.5-122B respectively, at comparable or better benchmark accuracy (8k input / 64k output)", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } } ], "assessment": { "value": "LatentMoE is the first atlas entry that computes routed experts in a compressed latent space — the same design family as Kimi K3's Stable LatentMoE (K3 cites the LatentMoE line, §2.3), so this is a shared new mechanism rather than a unique one, and Super's variant is the simpler un-stabilized version (no RMSNorm/SiTU-GLU stabilization described; activation function not even named). NVFP4 from-scratch pretraining over 25T tokens is a genuine first for the atlas (peers train BF16/FP8). The shared-weight MTP with recursive drafting is a real refinement over DeepSeek-V3's per-layer MTP heads. The rest is Nemotron lineage: Mamba-2 + GQA anchors (Jamba/Nemotron-H), aux-loss-free balancing (DeepSeek-V3), WSD + checkpoint merging (MiniCPM/WSM), GenRM RLHF (RLBFF), and the two-stage SFT loss is a small but sensible fix for long-input-short-output degradation. The 99.8%-accuracy NVFP4 PTQ plus the stochastic-rounding SSM cache analysis (recurrent quantization error accumulation) is a strong systems contribution.", "provenance": "inferred", "note": "comparison against atlas entries (jamba, deepseek-v3, mixtral, moonshot-kimi-k3, nvidia-nemotron-3-nano-30b-a3b) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all 88 layers carry LatentMoE FFNs ('sparse LatentMoE scaling', §2.1; MoE layers paired with Mamba-2 blocks throughout, §2.1.3)" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "per Figure 2, 7 of 22 layers in the repeating period contain attention (≈ 28/88 = 32% attention, 68% Mamba-2) — Mamba share below the 80% threshold, so hybrid rather than SSM" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights-open-data", "provenance": "derived", "note": "open_weights stated true; 'Nemotron 3 Super datasets, along with the base, post-trained, and quantized checkpoints, are open-sourced on HuggingFace' (Abstract); training recipe shared on the Nemotron Developer Repository (§1)" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "120.6B total params ≥ 100B threshold" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "title frames it as 'an Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning'; §1 states 'We trained Nemotron 3 Super with a strong emphasis on agentic capabilities'; three reasoning modes with budget control. Secondary intents: long-context (1M) and throughput-efficient serving, noted in prose." } }, "prose": { "design_overview": { "value": "Nemotron 3 Super 120B-A12B scales the Nemotron 3 hybrid recipe to 88 layers, 120.6B total / 12.7B active parameters (12.1B excluding embeddings), pairing a predominantly Mamba-2 backbone with periodically inserted GQA attention 'anchors' (32 query / 2 KV heads, head dimension 128) for full-token interaction — per Figure 2 a 22-layer period with seven attention-containing blocks repeats across the stack. Its headline innovation is LatentMoE: tokens are down-projected from 4096 into a 1024-wide latent space where 512 experts (top-22 per token, expert hidden dimension 2688) do all routed computation and all-to-all traffic, with expert count and top-k scaled up by d/ℓ = 4; the router, shared experts (intermediate size 5376), and non-expert layers stay in full width. Two shared-weight MTP layers add a multi-token-prediction objective (loss scale 0.3) and double as a native draft model for speculative decoding, reaching an average acceptance length of 3.45 on SPEED-Bench. The model is NoPE with untied embeddings and RMSNorm and — a family first — was pre-trained entirely in NVFP4 (final 15% of layers, latent projections, MTP, QKV/attention projections and embeddings in BF16; Mamba output projection MXFP8) stably across 25T tokens. Post-training is agentic-heavy: two-stage SFT (7M samples, 80B tokens) with a per-conversation normalized second stage, multi-environment RLVR over 21 environments, a separate end-to-end SWE-RL stage, PivotRL for long-horizon agentic domains, RLHF with a principle-following GenRM, and MTP healing. Deployment ships FP8 (Hopper) and NVFP4 (Blackwell) checkpoints with 99.8% median accuracy.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Hybrid Mamba-2 + GQA backbone with MoE FFNs", "provenance": "stated", "ref": "§1, §2.1 ('scales up the hybrid Mamba-Attention Mixture-of-Experts (MoE) architecture introduced in Nemotron-3 Nano'); Mamba-2 cited (Dao & Gu, 2024), §2.1.3; GQA named without citation in this report" }, "origin": { "value": "Nemotron 3 Nano / Nemotron-H / Jamba (Mamba-2); GQA (Ainslie et al., 2023)", "provenance": "stated", "ref": "§1, §2.1" } }, { "mechanism": { "value": "LatentMoE — experts operating in a compressed latent space", "provenance": "stated", "ref": "§2.1.1 (cites Elango et al., 2026 — LatentMoE technical report, not in atlas)" }, "origin": { "value": "LatentMoE (Elango et al., 2026, not in atlas)", "provenance": "stated", "ref": "§2.1.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) for quality and speculative decoding", "provenance": "stated", "ref": "§2.1.2 (cites Gloeckle et al., 2024; DeepSeek-AI, 2025c — DeepSeek-V3)" }, "origin": { "value": "DeepSeek-V3 / Gloeckle et al.", "provenance": "stated", "ref": "§2.1.2" } }, { "mechanism": { "value": "Aux-loss-free bias-based load balancing", "provenance": "stated", "ref": "§2.4 (cites Wang et al., 2024; DeepSeek-AI, 2025c)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.4" } }, { "mechanism": { "value": "Warmup-Stable-Decay LR schedule and checkpoint merging (weighted sliding-window averaging)", "provenance": "stated", "ref": "§2.4 (cites Hu et al., 2024); §2.5 (cites Wortsman et al., 2022; Tian et al., 2025 — WSM; Ling Team, 2025)" }, "origin": { "value": "MiniCPM (WSD) / Model Soups + WSM", "provenance": "stated", "ref": "§2.4, §2.5" } }, { "mechanism": { "value": "Principle-following generative reward model (GenRM) for RLHF", "provenance": "stated", "ref": "§3.2.3 (cites Wang et al., 2025b — RLBFF; Helpsteer 3 dataset (Wang et al., 2025c))" }, "origin": { "value": "RLBFF (Wang et al., 2025b, not in atlas)", "provenance": "stated", "ref": "§3.2.3" } }, { "mechanism": { "value": "PivotRL — assistant-turn-level RL reusing offline SFT expert trajectories at uncertain 'pivot' turns", "provenance": "stated", "ref": "§3.2.4 (cites Yi et al., 2026 — NVIDIA technical report, not in atlas)" }, "origin": { "value": "PivotRL (Yi et al., 2026, not in atlas)", "provenance": "stated", "ref": "§3.2.4" } } ], "influence_out": [ { "model_slug": { "value": "nvidia-nemotron-3-ultra-550b-a55b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Hybrid Mamba-Attention backbone — same architecture as Nemotron 3 Super (Mamba-2 layers + sparse global attention anchors)", "provenance": "inferred", "note": "mechanism as stated in nvidia-nemotron-3-ultra-550b-a55b report's lineage_in" } } ], "notable_omissions": { "value": [ "No vocabulary size or tokenizer algorithm disclosed", "No license named for the released weights", "Expert activation function never stated (Nemotron 3 Nano disclosed squared ReLU; Super is silent)", "Number of shared experts not disclosed (only the shared-expert intermediate size, 5376)", "Attention layer count not stated in the text — only the Figure 2 periodic pattern (7 attention-containing blocks per 22-layer period), whose exact repetition is left to the reader", "Main 25T-token pretraining hardware and parallelism not disclosed (only the LC-Phase: GB200, 64-way context / 2-way tensor / 64-way expert parallelism)", "No total training compute (FLOPs or GPU-hours) disclosed — only relative savings from checkpoint merging (~4T tokens ≈ 16% of the pretraining FLOP budget)", "Normalization placement (pre/post) and QK-norm not described", "Mamba SSM parameterization (discretization, decay initialization) not described despite Table 1 giving state dimension/groups/heads", "RLHF hyperparameters (prompts per step, generations per prompt, reward formula) deferred to the Nemotron 3 Nano recipe rather than restated" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "nvidia-nemotron-3-super-120b-a12b.json" }, { "slug": { "value": "nvidia-nemotron-3-ultra-550b-a55b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Ultra-Technical-Report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/nvidia-nemotron-3-ultra-550b-a55b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Nemotron 3 Ultra (Nemotron 3 family)", "provenance": "stated", "ref": "Title, Abstract" }, "variants": [], "org": { "value": "NVIDIA", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-06-09", "provenance": "stated", "ref": "Title page (first line of the PDF: '2026-6-9')" }, "report_url": { "value": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Ultra-Technical-Report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named in the report (release is via HuggingFace, but the report text never names a license)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We open-source the base, post-trained, and quantized checkpoints, along with the training data and recipe on HuggingFace'; §1 (checkpoint list: Base BF16, Post-Trained BF16, NVFP4, GenRM)" } }, "scale": { "total_params": { "value": 550, "provenance": "stated", "ref": "Abstract, Table 1, §6 (550B total)" }, "active_params_per_token": { "value": 55, "provenance": "stated", "ref": "Abstract, Table 1, §6 (55B active per token)" }, "layers": { "value": 108, "provenance": "stated", "ref": "Table 1 (Total Layers)" }, "hidden_dim": { "value": 8192, "provenance": "stated", "ref": "Table 1 (Model Dimension)" }, "ffn_inner_dim": { "value": 5120, "provenance": "stated", "ref": "Table 1 (Expert Hidden Dimension); note: per-expert hidden dim for LatentMoE routed experts; shared expert intermediate size is 10240; MoE latent size is 2048" }, "ffn_ratio": { "value": 0.625, "provenance": "derived", "note": "5120 / 8192 = 0.625 (per-expert inner dim over hidden dim, MoE)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 1 (Q-Heads n_q)" }, "attention_heads_kv": { "value": 2, "provenance": "stated", "ref": "Table 1 (KV-Heads n_kv) — GQA with 2 KV heads on the global attention layers" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 1 (Head Dimension)" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "vocabulary size never disclosed anywhere in the report" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed; embeddings are kept in higher precision than NVFP4 (Table 12) but tying is not addressed" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "every layer pairs a Mamba-2 or Attention block with a LatentMoE FFN (Figure 2 shows 'Latent MoE' in every block); the report calls it a Mixture-of-Experts hybrid Mamba-Attention model" }, "moe": { "expert_count": { "value": 512, "provenance": "stated", "ref": "Table 1 (Total Experts per Layer)", "note": "report prints 'Total Experts per Layer: 512'; it does not clarify whether the shared expert is included in this count (Shared Expert Intermediate Size is listed separately)" }, "experts_per_token": { "value": 22, "provenance": "stated", "ref": "Table 1 (Top-k / Activated Experts)" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "a shared expert clearly exists (Table 1 'Shared Expert Intermediate Size: 10240'; Table 12 row 'MoE shared experts') but the report never states how many shared experts per layer" }, "routing": { "value": null, "provenance": "unknown", "note": "router function not described in this report (details deferred to the LatentMoE citation, Elango et al., 2026, and the Nemotron 3 Super report)" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "not described; MaxVio is used only as a training-health monitor (§2.7), not as a stated balancing mechanism" }, "expert_granularity": { "value": "fine-grained LatentMoE: routed experts operate in a compact latent space (MoE Latent Size 2048) rather than at full hidden width; 512 experts per layer with top-22 active", "provenance": "inferred", "note": "evidence: Table 1 (MoE Latent Size 2048), §2.1 (LatentMoE cited, Elango et al., 2026), §5 ('LatentMoE buys more routed experts at fixed inference cost by trading away hidden-dimension width')" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.1, §5 ('hybrid Mamba-2 stack with sparse global Attention anchors'); Table 1 (Q-Heads 64, KV-Heads 2, Head Dim 128 — the attention layers are GQA)" }, "attention_layer_pattern": { "value": "sparse global Attention layers anchored among Mamba-2 layers: repeating groups of Mamba-2 + LatentMoE blocks with periodic Attention + LatentMoE anchors (Figure 2); exact per-layer counts are not stated in the prose", "provenance": "stated", "ref": "Figure 2, §2.1, §5; note: the PDF text extraction of Figure 2 is partially garbled ('x3/x4/x2' multipliers), so exact attention-layer count is not recoverable from the stored text" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no learned attention over prior layer outputs is described; standard residual stack (residual-norm dynamics discussed in §2.7 imply plain residual connections)" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "positional encoding never mentioned in the report (Mamba-2 layers are recurrent and need none; the attention layers' scheme is not stated)" }, "rope_base": { "value": null, "provenance": "unknown", "note": "no RoPE configuration disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "no RoPE configuration disclosed" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "context is extended by LC-Phase continuous pretraining at 1M (§2.5) with no positional-encoding modification disclosed" }, "factor": { "value": null, "provenance": "unknown", "note": "not applicable as stated — no PE scaling method disclosed" }, "target_length": { "value": 1048576, "provenance": "stated", "ref": "§2.5 (CPT on 1,048,576 (1M) context length for 92% of iterations)" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "normalization type never named in the report" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post placement not described" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "QK-normalisation on the global-attention layers is never described" } }, "activation": { "value": null, "provenance": "unknown", "note": "activation function never named (LatentMoE / Mamba-2 internals not described in this report)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink mechanism described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no attention-logit softcapping described" }, "other": [ { "value": "FP32 gradient-reduction recipe for the output layer: a switch to BF16 local gradient accumulation caused training divergence 1 at ~8T tokens (MTP wgrad contribution to the shared output layer was lost in BF16); rolling back to full FP32 reduction re-stabilized training", "provenance": "stated", "ref": "§2.7 (Divergence 1)" }, { "value": "Early LR annealing: divergence 2 at ~16T tokens was mitigated by rewinding to the 15T checkpoint and starting annealing early; the pretraining horizon was cut to 20T tokens", "provenance": "stated", "ref": "§2.7 (Divergence 2)" }, { "value": "MaxVio expert-imbalance monitoring: peak-expert load metric (max attainable 23.27 for 512/22) tracked as a training-health proxy; routing skew and 'dead experts' correlated with the divergence", "provenance": "stated", "ref": "§2.7" }, { "value": "Residual-stream activation norm monitoring: early-layer residual norms spiked from ~7.5T tokens onward (4 orders of magnitude spread across depth), used as an instability signal", "provenance": "stated", "ref": "§2.7, Figure 8" } ] } }, "context": { "trained_length": { "value": 1048576, "provenance": "stated", "ref": "§2.5 (LC-Phase CPT at 1,048,576 (1M) context for 92% of iterations, 4,096 for the remaining 8%; 33B tokens total)" }, "deployed_length": { "value": 1000000, "provenance": "stated", "ref": "Abstract, §3.7.2 ('supports contexts of up to 1M tokens')" }, "extension_method": { "value": "LC-Phase continuous pretraining (CPT): 1M context for 92% of iterations and 4K for 8% (math/code SFT-style data only), constant LR 2.5e-6, 33B tokens; long-context data 46% + Phase-2 data 54% in the blend; no RULER-style data", "provenance": "stated", "ref": "§2.5" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer never described in the report" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 20, "provenance": "stated", "ref": "Abstract, §2.4 ('total horizon of 20 trillion tokens')" }, "data_composition": { "value": "Two-phase mixture (adaptation of the Nemotron 3 Super/Nano recipe): phase 1 (~15T tokens, ~75%) biases diversity, phase 2 (~5T) biases quality; 19 high-level categories, largest being quality-filtered and synthetic web crawl (~49% phase 1 / ~38% phase 2: crawl-medium/high, syn-crawl-medium/high), code 14%, math 6.4%, multilingual 5% (11 languages), nemotron-cc-code 2.1%, finepdfs (upweighted in phase 2), wiki, academic, legal, Crawl++, and synthetic SFT-style data (sft-code/stem/general); new datasets added since Super: Nemotron-Pretraining-Code-v3 (173B fresh GitHub tokens, cut-off Sep 30 2025), benchmark-seeded synthetic Multiple-Choice and Generative QA, Fact-Seeking (from Finewiki), Moral-Scenarios (CoT via Qwen3-235B), and a Legal family (regulations, case-law summaries, CaseHOLD/ContractNLI reformats, synthetic outcome/definition/diversity questions)", "provenance": "stated", "ref": "§2.3, Figure 4" }, "curriculum": { "value": "Two-phase pretraining (15T diversity-biased then 5T quality-biased, transition at ~75% of pretraining) under a Warmup-Stable-Decay LR schedule; offline checkpoint merging (sliding 500B-token merge window, 25B checkpoint interval) for evaluation; final checkpoint selected over a large merge sweep; followed by the 1M LC-Phase CPT (33B tokens)", "provenance": "stated", "ref": "§2.3.6, §2.4, §2.5" }, "optimizer": { "value": null, "provenance": "unknown", "note": "optimizer never named in the report" }, "lr_schedule": { "value": "Warmup-Stable-Decay (WSD): 200B-token warmup to peak 2.5e-4, minus-sqrt decay over the final 5T tokens to 2.5e-6; MTP loss scaling factor 0.1 (0.05 per MTP block)", "provenance": "stated", "ref": "§2.4" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "pretraining batch sizes not disclosed (SFT stage-1/2 global batch 64 and RL global batch 8192 are post-training)" }, "precision": { "value": "NVFP4 pre-training (E2M1 datatype with two-dimensional block quantization on weights, Random Hadamard Transforms on wgrad inputs, stochastic rounding on gradients); the final 15% of the network (16 layers), Mamba output projections, latent projections, QKV and attention projections, MTP layers, and embedding layers kept in higher precision; claimed largest-scale stable NVFP4 training to date", "provenance": "stated", "ref": "§2.2" }, "parallelism": { "value": "LC-Phase: 32-way context parallelism, 8-way tensor parallelism, 128-way expert parallelism, 2-way pipeline parallelism; post-training RL on GB200 with Megatron EP groups co-located in one NVLink domain (NVLink-aware rank placement)", "provenance": "stated", "ref": "§2.5, §3.6.2" }, "hardware": { "value": "GB200 GPUs (LC-Phase 'train on GB200 GPUs'; production RL cluster on GB200 NVL72 nodes)", "provenance": "stated", "ref": "§2.5, §3.6.2" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (two-stage SFT: stage 1 packed 294,912-token sequences, global batch 64, 204,800 samples, cosine LR peak 1.5e-5; stage 2 packed 515,000-token sequences with long-context data up to 512K, 19,200 samples; shared-weight MTP objective retained)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§3.2 (unified RLVR 'largely follows the asynchronous GRPO algorithm with the stability optimizations proposed in NVIDIA (2026)'; global batch 8192, 16 rollouts/sample, max generation 48K→64K tokens); §3.3.2 (Chat teacher additionally uses RLHF with an Ultra-based generative reward model / GenRM)" }, "reasoning_training": { "value": "Unified RLVR across terminal usage, office/productivity, software engineering, search, tool calling, math, code, STEM, safety, chat, instruction following, long-context QA, and reasoning environments; three reasoning modes (reasoning-off, regular, medium-effort) with inference-time budget control (medium-effort ~2.5× fewer tokens at ~7% accuracy cost); >10 domain-specialized teachers; test-time scaling via generate-verify-refine on Olympiad math (IMO-ProofBench Advanced 82.3%)", "provenance": "stated", "ref": "§3.2, §3.3, §3.5, §3.7.3" }, "distillation": { "value": "Multi-teacher On-Policy Distillation (MOPD): >10 domain-specialized teachers (STEM/general reasoning, competitive coding, SWE, office & workplace, search, terminal-use, conversational tool-use, usability, agentic safety, chat, instruction-following & factuality) consolidated into the student over two iterative cycles via asynchronous dense token-level supervision (negative reverse-KL on student-generated rollouts, PPO-style clipping, IcePop masking), with a light SFT warmup to align student rollouts with teacher distributions; MTP Boosting additionally aligns the draft head to backbone logits via temperature-scaled forward-KL", "provenance": "stated", "ref": "§3.3, §3.3.1, §3.4" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: the report describes only text/code/terminal/tool data and evaluations; no vision or other modality appears anywhere" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type = text-only" } }, "efficiency": { "kv_cache": { "value": "hybrid cache: Mamba-2 layers keep a constant-size SSM state per sequence (FP32 at train, FP16 with stochastic rounding at deploy; block-scaled INT8 explored); global-attention layers keep a small GQA KV cache (2 KV heads, FP8 at deploy); the 32-bit Mamba cache is larger than the FP8 KV cache at sequence lengths up to 64K", "provenance": "stated", "ref": "§4.5, Table 12, §5.1" }, "quantization": { "value": "Single NVFP4 checkpoint at 5.03 bits-per-element: NVFP4 routed-expert GEMMs (E2M1, dynamic max-based activation scaling, Four-Over-Six 4/6 weight scaling), FP8 per-tensor shared experts and Mamba linear layers, BF16 attention linear layers / MoE latent projections / embeddings / MTP layers, FP8 KV cache, FP16-SR Mamba cache; runs native W4A4 on Blackwell and W4A16 on Hopper", "provenance": "stated", "ref": "§4.1, §4.3, §4.6, Table 12" }, "speculative_dedup": { "value": "Native Multi-Token Prediction (MTP): two shared-weight MTP blocks (each a single attention layer + single MoE layer) trained throughout pretraining/SFT/RL as an internal drafter; MTP Boosting (head-only forward-KL distillation, temperature 2, 7 MTP steps) fixes the train-inference mismatch; draft length is a deployment knob (peak 2.89× decode speedup at DL=6); SPEED-Bench average acceptance length 4.584 (greedy, boosted)", "provenance": "stated", "ref": "§2.1, §3.4, Table 6, §5.1" }, "serving": { "value": "TRT-LLM for the headline throughput numbers (vLLM for baselines); wide EP for high-throughput / wide TP for low-latency serving on GB200 NVL72 (single NVLink domain); prefill-decode disaggregation for hybrid Mamba-Attention landed in vLLM (~10% throughput gain); FlashInfer NVLinkOneSided all-to-all backend (~5% gain) and DWDP weight-prefetch alternative; EPLB hot-expert replication; MoE-side prefill chunking; SSM-state snapshotting at every draft step for speculative rollback and coarse-grained prefix caching", "provenance": "stated", "ref": "§5.2, §5.3" } }, "evaluation": { "benchmarks": [ { "name": { "value": "Terminal Bench 2.1", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 56.4, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "SWE-Bench Verified", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 70.7, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "GDPVal", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 46.7, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "TauBench V3 (Average)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 70.9, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 44.4, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "PinchBench", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 90.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "ProfBench (Search)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 56.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "IOI 2025 (Score / 600)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 570.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "LiveCodeBench (v6)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 89.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "IMOAnswerBench (with tools)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 92.3, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "GPQA (no tools)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 87.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 86.8, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "HLE (no tools)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 26.7, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 65.4, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "RULER (1M)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 94.7, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "MMLU-ProX (avg 10 langs)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 83.0, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "IFBench (prompt loose)", "provenance": "stated", "ref": "Table 10" }, "value": { "value": 81.7, "provenance": "stated", "ref": "Table 10" }, "ref": { "value": "Table 10", "provenance": "stated", "ref": "Table 10" } }, { "name": { "value": "MMLU (base, 5-shot acc)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 89.08, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "RULER 64K (base)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 95.3, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "IMO-ProofBench Advanced (test-time scaling)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": "82.3% (173/210)", "provenance": "stated", "ref": "Table 11" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } } ] }, "contributions": { "claimed": [ { "text": { "value": "An open, efficient 550B-total / 55B-active MoE Hybrid Mamba-Attention model achieving up to ~6× higher inference throughput than state-of-the-art open LLMs (5.9× vs GLM-5.1-754B-A40B, 4.8× vs Kimi-K2.6-1T-A32B, 1.6× vs Qwen-3.5-397B-17B at 8K/64K) at on-par accuracy, with a 1M-token context for long-running agentic tasks", "provenance": "stated", "ref": "Abstract, §1, Figure 1" }, "ref": { "value": "Abstract, §1, Figure 1", "provenance": "stated", "ref": "Abstract, §1, Figure 1" } }, { "text": { "value": "LatentMoE for better accuracy per parameter than standard Granular MoEs, buying more routed experts at fixed inference cost by trading away hidden-dimension width", "provenance": "stated", "ref": "§1, §2.1, §5" }, "ref": { "value": "§1, §2.1, §5", "provenance": "stated", "ref": "§1, §2.1, §5" } }, { "text": { "value": "Native Multi-Token Prediction with two shared-weight heads used for speculative decoding throughout training, plus MTP Boosting (head-only forward-KL distillation) that closes the train-inference mismatch of the draft head", "provenance": "stated", "ref": "§2.1, §3.4" }, "ref": { "value": "§2.1, §3.4", "provenance": "stated", "ref": "§2.1, §3.4" } }, { "text": { "value": "The largest-scale demonstration of stable and accurate NVFP4 pre-training to date (20T tokens, E2M1 with 2D block quantization, <0.4% relative train-loss gap vs BF16 segments)", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "Agent-focused post-training: unified multi-environment RLVR followed by two iterations of asynchronous Multi-teacher On-Policy Distillation from more than ten domain-specialized teachers, with reasoning-effort control (reasoning-off / regular / medium-effort) and inference-time budget control", "provenance": "stated", "ref": "§3.2, §3.3, §3.5" }, "ref": { "value": "§3.2, §3.3, §3.5", "provenance": "stated", "ref": "§3.2, §3.3, §3.5" } }, { "text": { "value": "1M-token context via an LC-Phase continuous-pretraining recipe (92% 1M / 8% 4K iterations, 33B tokens) that preserves short-benchmark accuracy", "provenance": "stated", "ref": "§2.5" }, "ref": { "value": "§2.5", "provenance": "stated", "ref": "§2.5" } }, { "text": { "value": "Inference infrastructure for hybrid Mamba-Attention serving at scale: prefill-decode disaggregation with hybrid cache-state transfer upstreamed to vLLM, FlashInfer one-sided all-to-all, DWDP, EPLB hot-expert replication, MoE-side prefill chunking, and SSM-state snapshotting for draft rollback and prefix caching", "provenance": "stated", "ref": "§5.2, §5.3" }, "ref": { "value": "§5.2, §5.3", "provenance": "stated", "ref": "§5.2, §5.3" } }, { "text": { "value": "Full openness: base, post-trained, and NVFP4-quantized checkpoints plus training data, recipes, and RL environments released on HuggingFace", "provenance": "stated", "ref": "Abstract, §1, §6" }, "ref": { "value": "Abstract, §1, §6", "provenance": "stated", "ref": "Abstract, §1, §6" } } ], "assessment": { "value": "The block is a scaled-up Nemotron 3 Super: hybrid Mamba-2 + sparse GQA attention with LatentMoE FFNs, so within the atlas its closest relatives are ai21-jamba (hybrid SSM-attention, dense) and deepseek-v3 (frontier MoE + MTP) rather than any all-attention MoE. Biggest design differences vs deepseek-v3: the Mamba-2 majority backbone (attention only as sparse anchors, 2 KV heads) instead of all-MLA layers, NVFP4 pre-training instead of BF16/FP8, and the two-iteration MOPD pipeline instead of single-model RLVR. MTP-as-native-drafter and NVFP4 5.03-BPE mixed quantization are shared with the Nemotron 3 family rather than novel vs the atlas; MOPD (on-policy distillation of multiple teachers) mirrors the kimi-k3 and nemotron-cascade-2 direction and is not new to the atlas. The 550B/55B scale with a 512-expert LatentMoE (22 active) and 1M context is the largest hybrid-SSM record in the atlas. Claims of ~6× throughput are measured in-house on GB200 (TRT-LLM vs vLLM baselines) and are not independently verifiable.", "provenance": "inferred", "note": "comparison against atlas entries (ai21-jamba, deepseek-v3, moonshot-kimi-k3, mistral-mixtral-8x7b) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "every layer carries a LatentMoE FFN (Figure 2); the report itself calls it a Mixture-of-Experts model" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "hybrid Mamba-2 stack with sparse global-attention anchors (§5); the report explicitly names the architecture 'Hybrid Mamba-Attention' (title, §2.1); exact Mamba:attention layer ratio is in Figure 2 only (garbled in extraction) but attention is explicitly described as sparse/minority" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights-open-data", "provenance": "derived", "note": "open_weights stated true; the report additionally states the training data and recipes are open-sourced on HuggingFace (Abstract, §1, §6)" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "550B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as NVIDIA's most capable general open model with a broad agentic/reasoning/knowledge/chat evaluation suite; primary emphasis is long-running autonomous agentic tasks and inference efficiency ('ideal for long-running autonomous agentic tasks' — Abstract); secondary intents: agentic-reasoning and long-context (1M), noted in prose" } }, "prose": { "design_overview": { "value": "Nemotron 3 Ultra is a 108-layer, 550B-parameter sparse MoE (55B active per token) built as a scaled-up Nemotron 3 Super: a hybrid Mamba-Attention backbone in which Mamba-2 state-space layers dominate and sparse global-attention anchors (GQA, 64 query heads / 2 KV heads, head dim 128) provide periodic exact-attention interaction, with every layer carrying a LatentMoE FFN (512 experts per layer, top-22 activated, experts operating in a 2,048-wide latent space instead of the full 8,192 width, plus a shared expert with 10,240 intermediate size). Pre-training runs 20T tokens in NVFP4 — E2M1 weights with 2D block quantization and stochastic rounding, with the final 16 layers and all attention/latent/embedding/MTP projections held in higher precision — under a Warmup-Stable-Decay schedule (200B warmup to 2.5e-4, minus-sqrt decay over the final 5T), split into a 15T diversity phase and a 5T quality phase. Two training divergences are dissected in detail: a BF16 gradient-accumulation regression for the output layer (fixed by returning to FP32 reduction) and an undetermined divergence at ~16T (mitigated by early annealing to a 20T horizon). A 33B-token LC-Phase continuous-pretraining stage (92% of iterations at 1,048,576 tokens, 8% at 4K) delivers the 1M context. Post-training is agent-focused: two-stage SFT, unified RLVR, then two iterations of asynchronous Multi-teacher On-Policy Distillation from more than ten specialized teachers (SWE, office, search, terminal-use, chat, safety, STEM, coding…), finished with MTP Boosting that aligns the shared-weight two-block draft head to backbone logits for speculative decoding. Reasoning-effort control (off/regular/medium) tunes the accuracy-compute trade-off at inference.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "LatentMoE — routed experts operating in a compact latent space", "provenance": "stated", "ref": "§2.1 (cites Elango et al., 2026, arXiv:2601.18089)" }, "origin": { "value": "LatentMoE (not in atlas)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Hybrid Mamba-Attention backbone — same architecture as Nemotron 3 Super (Mamba-2 layers + sparse global attention anchors)", "provenance": "stated", "ref": "§2.1 ('same hybrid Mamba-Attention MoE architecture as Nemotron 3 Super')" }, "origin": { "value": "Nemotron 3 Super (not in atlas)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) — shared-weight auxiliary heads doubling as a speculative draft model", "provenance": "stated", "ref": "§2.1 (cites Gloeckle et al., 2024)" }, "origin": { "value": "Gloeckle et al. (not in atlas)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "On-policy distillation of specialized teachers into one student (MOPD builds on Nemotron-Cascade-2 and Thinking Machines' on-policy distillation)", "provenance": "stated", "ref": "§3.3.1 (cites Yang et al., 2026; Lu & Lab, 2025; Xiao et al., 2026)" }, "origin": { "value": "Nemotron-Cascade-2 (not in atlas)", "provenance": "stated", "ref": "§3.3.1" } } ], "influence_out": [], "notable_omissions": { "value": [ "Vocabulary size and tokenizer never disclosed", "Normalization type/placement and activation function never named (LatentMoE / Mamba-2 internals deferred to the LatentMoE and Super reports)", "MoE router function and load-balancing mechanism not described in this report (only MaxVio monitoring)", "Shared-expert count not stated (Table 1 prints 'Total Experts per Layer: 512' without clarifying inclusion of the shared expert)", "Pretraining optimizer and batch sizes not disclosed", "No training compute (FLOPs/GPU-hours) disclosed", "No license named", "Input/output embedding tying not disclosed", "Exact attention-vs-Mamba layer counts appear only in Figure 2 and are not stated in prose (PDF extraction of the figure is garbled)", "RoPE / positional-encoding configuration of the attention layers not disclosed", "Internal inconsistency: GLM-5.1 is '754B-A40B' in §1 but '744B-A40B' in Table 10", "Total parameters (550B) vs active (55B) are stated but no per-component breakdown (attention + Mamba + routed/shared experts + embeddings) is given to verify the active count" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "nvidia-nemotron-3-ultra-550b-a55b.json" }, { "slug": { "value": "nxai-xlstm-7b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2503.13427", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/nxai-xlstm-7b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "xLSTM-7B", "provenance": "stated", "ref": "Title ('xLSTM 7B: A Recurrent LLM for Fast and Efficient Inference')" }, "variants": [], "org": { "value": "NXAI (NX-AI)", "provenance": "stated", "ref": "Title page byline; model/code links (huggingface.co/NX-AI/xLSTM-7b, github.com/NX-AI)" }, "release_date": { "value": "2025-03-17", "provenance": "inferred", "note": "Report text is undated (ICML submission format); arXiv v1 submission date 2025-03-17 per export.arxiv.org API metadata (arXiv:2503.13427)" }, "report_url": { "value": "https://arxiv.org/abs/2503.13427", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "report says weights/code are open-source but never names a license" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('Our model weights, model code and training code are open-source'); §1 ('We release the pre-trained model xLSTM 7B on Huggingface')" } }, "scale": { "total_params": { "value": 6.87, "provenance": "stated", "ref": "Table 4 (Num Params 6,865,424,896 ≈ 6.87B); Abstract ('7-billion-parameter LLM')" }, "active_params_per_token": { "value": 6.87, "provenance": "derived", "note": "dense recurrent model: active params = total params (6.87B); no routing or sparsity anywhere" }, "layers": { "value": 32, "provenance": "stated", "ref": "Table 4 (Num Blocks 32); Appendix A ('32 post-up projection blocks')" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 4 (Model Dim 4096); Table 3 ('embedding dimension 4096')" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "SwiGLU projection factor 2.66 disclosed (Fig. 8) but the exact inner dimension is never printed; 4096 × 2.66 = 10,895.36 is non-integer" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: ffn_inner_dim not disclosed" }, "attention_heads_q": { "value": 8, "provenance": "stated", "ref": "Table 4 (Num Heads 8; multi-head mLSTM, §2.1)" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§2 (mLSTM cells keep a fixed-size recurrent matrix state (C_t, n_t, m_t) per head; there are no KV heads and no KV cache)" }, "head_dim": { "value": 512, "provenance": "derived", "note": "hidden_dim 4096 / attention_heads_q 8 = 512 (integer); matches Table 3's d_hv = 512 row for the 8-head configuration and Eq. d = #heads × d_hv (§2.1, Fig. 8)" }, "vocab_size": { "value": 50257, "provenance": "stated", "ref": "Table 4 (Vocab Size 50257); Appendix A ('vocabulary size 50257')" }, "embedding_tied": { "value": false, "provenance": "stated", "ref": "Appendix A ('we do not tie the weights for input layers (embedding) and output layers (logits)')" }, "variants": [] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "all 32 blocks are dense mLSTM + SwiGLU blocks; no MoE anywhere (Appendix A, Fig. 8)" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§2 (no MoE in the architecture)" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§2 (no MoE in the architecture)" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§2 (no MoE in the architecture)" }, "routing": { "value": null, "provenance": "n/a", "ref": "§2 (no MoE in the architecture)" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§2 (no MoE in the architecture)" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§2 (no MoE in the architecture)" } }, "attention_variant": { "value": "linear/state-space", "provenance": "stated", "ref": "§2 (mLSTM recurrent cell with matrix memory: recurrent mode for generation, fully parallelizable mode for training; constant memory, linear compute in sequence length)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "32 identical post-up projection blocks, each an output-gated mLSTM sequence-mix layer plus a SwiGLU channel-mix layer (§3.1, Fig. 8); no attention layers of any kind" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: standard residual stacked blocks (§3, Fig. 8); no attention-over-layers mechanism mentioned" }, "position_encoding": { "method": { "value": "NoPE", "provenance": "inferred", "note": "report never describes any positional encoding; order is carried implicitly by the recurrent mLSTM state (architecture described in detail, §2–3)" }, "rope_base": { "value": null, "provenance": "n/a", "ref": "report-wide absence (no RoPE anywhere in the model)" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "report-wide absence (no RoPE)" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "no context-extension mechanism: the standard model is trained at 8K and used at 8K (§1, App. B); the experimental LCTX variant reaches 32K via a long-context cool-down data stage, not a positional-encoding extension (§5.1)" }, "factor": { "value": null, "provenance": "n/a", "ref": "no extension mechanism (see extension.method)" }, "target_length": { "value": null, "provenance": "n/a", "ref": "no extension mechanism (see extension.method)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "Fig. 8 ('The Norm layers are RMS norms (Zhang & Sennrich, 2019), the Headwise Norm is a Layernorm (Ba et al., 2016)'); §3.2 ('we replace the LayerNorm by RMSNorm in our xLSTM architecture')" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§3.2 ('pre-norm at the block entry', 'in the pre-norm setting'); §5.3 ('two normalization layers, a pre-norm at the block entry and a head-wise norm layer after the mLSTM cell')" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "QK-Norm is a softmax-attention concept; the model has no softmax-attention layers, and any query/key normalization inside the mLSTM cell would be part of the linear-attention parameterization, not QK-Norm. Report does not discuss QK-Norm." } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "Fig. 8 ('the upper part is a Gated MLP (SwiGLU) as a feature/channel-mix layer'); §3.1 (projection factor 2.66)" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "report silent; attention sinks are a softmax-attention concept and the model is fully recurrent" }, "softcapping": { "value": 30, "provenance": "stated", "ref": "§3.2 Eq. 13 ('we apply soft-capping with a=30 to the final layer logits, similar to Gemma Team (2024b)')" }, "other": [ { "value": "Gate soft-capping of input and forget gate pre-activations with a=15 (softcap_a(x) = a·tanh(x/a)) to bound outlier features and loss spikes", "provenance": "stated", "ref": "§3.2 Eq. 13" }, { "value": "Negative input gate bias initialization (-10) to mitigate gradient-norm spikes early in training", "provenance": "stated", "ref": "§3.2, §5.3" }, { "value": "RMSNorm (instead of LayerNorm) for the pre-norm at block entry — strong stabilizing effect at 7B scale", "provenance": "stated", "ref": "§3.2, §5.3" }, { "value": "Memory-state reset at EOD token borders during sequence packing (forget gate forced to zero, memory reset to zero matrix) to prevent cross-document information leakage", "provenance": "stated", "ref": "App. B (Sequence packing)" }, { "value": "Gradient clipping norm 0.5", "provenance": "stated", "ref": "App. B (Optimization)" } ] } }, "context": { "trained_length": { "value": 8192, "provenance": "stated", "ref": "§1 ('trained on 2.3T tokens from the DCLM dataset with context length 8192'); App. B ('batch size 512 and context length 8192')" }, "deployed_length": { "value": 8192, "provenance": "derived", "note": "no context extension described for the standard model: trained and evaluated at 8K (§1, App. B). The experimental LCTX variant (not released) trains at 32K during the cool-down stage (§5.1)" }, "extension_method": { "value": "none", "provenance": "inferred", "note": "no extension mechanism disclosed for the standard model (trained and used at 8K). Context growth appears only in the experimental LCTX variant via a long-context cool-down data stage at 32K (§5.1), not a positional-encoding extension" } }, "tokenizer": { "algorithm": { "value": "BPE (GPT-NeoX-20B tokenizer)", "provenance": "stated", "ref": "Appendix A ('We use the GPT-NeoX-20B tokenizer (Black et al., 2022) with vocabulary size 50257')" }, "notes": { "value": "GPT-NeoX-20B BPE tokenizer, vocabulary 50,257; input (embedding) and output (logits) weights not tied; EOD token used for sequence packing with mLSTM memory-state reset at document borders", "provenance": "stated", "ref": "Appendix A, Appendix B" } }, "training": { "tokens": { "value": 2.3, "provenance": "stated", "ref": "§1, App. B ('a total of 2.3T (trillion) training tokens'; 550K steps × batch 512 × context 8192)" }, "data_composition": { "value": "Two-stage recipe using only publicly available datasets. Stage 1 (500K steps): exclusively the DCLM dataset. Stage 2 (50K steps, prioritizing math/coding/Q&A): DCLM 40%, FineWeb-Edu 15%, ProofPile-2 15%, TheStack 15%, Cosmopedia 10%, and SFT Q&A datasets 5% (NuminaMath CoT, MetaMathQA, Tulu v3.1, OpenHermes 2.5, GSM8K, Smoltalk subsets magpie-ultra/longalign/self-oss-instruct)", "provenance": "stated", "ref": "App. B (Table 5)" }, "curriculum": { "value": "two-stage: 500K steps on DCLM, then 50K steps on a math/coding/Q&A-heavy mixture including small SFT Q&A datasets; batch-size ramp-up (128 → 256 → 512); sequence packing with EOD-token memory-state reset; the experimental LCTX variant replaces the final cool-down with a 32K-context stage (batch 128, long-context corpora)", "provenance": "stated", "ref": "App. B" }, "optimizer": { "value": "AdamW with peak learning rate 5×10⁻⁴, β1=0.99, β2=0.95, ε=1e-8, weight decay 0.1, gradient clipping norm 0.5", "provenance": "stated", "ref": "App. B (Optimization)" }, "lr_schedule": { "value": "linear warm-up over 3,000 steps, exponential decay over 540,000 steps (decay factor chosen so 0.1× peak is reached after 500,000 steps), linear cool-down over 7,000 steps to 0", "provenance": "stated", "ref": "App. B (Optimization)" }, "batch_schedule": { "value": "batch size ramp-up: 128 for the first 2,000 steps, 256 for the next 2,000 steps, full batch size 512 afterward; LCTX variant reduces batch to 128 at 32K context to keep tokens-per-batch constant", "provenance": "stated", "ref": "App. B" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision (BF16/FP32/FP8) not disclosed" }, "parallelism": { "value": "Fully Sharded Data Parallel (FSDP) with activation checkpointing", "provenance": "stated", "ref": "App. B (Optimization)" }, "hardware": { "value": "128 NVIDIA H100 GPUs", "provenance": "stated", "ref": "§1, App. B" }, "compute": { "value": null, "provenance": "unknown", "note": "no total FLOPs or GPU-hours disclosed (only relative training-step speedups, e.g. 3.5× vs the previous xLSTM block, Table 2)" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "App. B ('the second training stage includes a collection of small supervised fine-tuning (SFT) Q&A datasets'); note: this is a training-recipe stage, not a chat-model post-training pipeline — only the pre-trained base model is released (§1)" }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "reasoning_training": { "value": "none disclosed", "provenance": "inferred", "note": "no reasoning RL, CoT post-training, or test-time-compute training described; the model is positioned for test-time compute scaling by users (§1), not trained for it" }, "distillation": { "value": "none", "provenance": "stated", "ref": "§1 (model trained from scratch on 2.3T tokens from DCLM; no distillation mentioned)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes a text-only language model; no vision or multimodal components anywhere" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1 (text-only architecture)" } }, "efficiency": { "kv_cache": { "value": "recurrent mLSTM state: constant-memory fixed-size state per head (C_t, n_t, m_t) instead of a growing KV cache — 8 heads × d_hv 512 = 134.2 MB total memory state, equivalent to a 128-token KV cache in a similar-sized Transformer (Table 3)", "provenance": "derived", "note": "from attention_variant linear/state-space; state-size numbers stated in Table 3 (8-head row: 134.2 MB, 128 KV-cache-token equivalent)" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence (inference speed benchmarks optimize with torch.compile and CUDA graphs, §5.2, but no serving-system optimizations are disclosed)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "BBH (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.381, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "MMLU-Pro (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.242, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "Math (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.036, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "MuSR (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.379, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "GPQA (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.28, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "IFEval (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.244, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "Average (Open LLM Leaderboard v2)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 0.26, "provenance": "stated", "ref": "Table 1 (xLSTM-7B)" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "ARC-C (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.574, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMLU (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.578, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "HellaSwag (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.714, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Winogrande (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.738, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "TruthfulQA (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.419, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "OpenBookQA (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.448, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "PiQA (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.819, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Average (Open LLM Leaderboard v1)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 0.613, "provenance": "stated", "ref": "Table 6 (xLSTM-7B)" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } } ] }, "contributions": { "claimed": [ { "text": { "value": "xLSTM-7B: the first xLSTM-based LLM scaled to modern data budgets (7B parameters, 2.3T tokens), showing the architecture matches similar-sized Transformers and Mamba models on downstream tasks", "provenance": "stated", "ref": "§1, §5.1" }, "ref": { "value": "§1, §5.1", "provenance": "stated", "ref": "§1, §5.1" } }, { "text": { "value": "Post-up projection block redesign: mLSTM operated in the embedding dimension (instead of a higher-dimensional up-projected space) with position-wise SwiGLU MLPs after each mLSTM layer, discarding channel-wise convolutions and learnable skip connections — 2×–4× higher token throughput than the previous xLSTM architecture with similar language modeling performance", "provenance": "stated", "ref": "§1, §3.1" }, "ref": { "value": "§1, §3.1", "provenance": "stated", "ref": "§1, §3.1" } }, { "text": { "value": "Stability optimizations for the mLSTM gating mechanism at scale: RMSNorm pre-norm, soft-capping of input/forget gates (a=15), negative input-gate bias initialization, and final-logit soft-capping (a=30)", "provenance": "stated", "ref": "§1, §3.2" }, "ref": { "value": "§1, §3.2", "provenance": "stated", "ref": "§1, §3.2" } }, { "text": { "value": "Fastest and most efficient 7B LLM in their benchmarks: about 50% faster text generation than Mamba, highest prefill throughput (about 70% higher than Codestral Mamba), lowest GPU memory footprint, and fastest time-to-first-token at batch size 1", "provenance": "stated", "ref": "§1, §5.2" }, "ref": { "value": "§1, §5.2", "provenance": "stated", "ref": "§1, §5.2" } }, { "text": { "value": "Long-context cool-down training enables strong RULER performance (20% average accuracy at 131K for the LCTX variant) despite training at only 32K context during the cool-down, outperforming RWKV-5/6 and matching state-space baselines", "provenance": "stated", "ref": "§5.1" }, "ref": { "value": "§5.1", "provenance": "stated", "ref": "§5.1" } } ], "assessment": { "value": "The first pure-recurrent (zero softmax-attention) record in the atlas. Against atlas peers: the mLSTM cell itself is inherited unchanged from Beck et al. (2024) xLSTM; the genuine architectural contribution is the post-up projection block — mLSTM run at the embedding dimension with a separate SwiGLU MLP, and removal of convolutions/learnable skips — which is a real efficiency refinement, not a new mixing family. The stability toolkit (RMSNorm pre-norm, gate soft-capping, negative gate-bias init, logit soft-capping a=30) adapts Transformer/Mamba practice (Gemma-style softcapping explicitly cited). Notably the scalar sLSTM branch of the original xLSTM is dropped entirely and never mentioned. The reported efficiency gains (≈50% faster generation than Mamba, ≈70% higher prefill throughput than Codestral Mamba) are internal single-GPU benchmarks with torch.compile/CUDA-graph setups and are not independently verified here. Downstream performance is mid-range among 7B models (Table 1), which the authors attribute to dataset quality.", "provenance": "inferred", "note": "comparison against atlas entries (ai21-jamba, moonshot-kimi-linear-48b-a3b, meta-llama-3, qwen-qwen3-dense) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "32/32 blocks are dense mLSTM + SwiGLU blocks; no routing or MoE anywhere (Appendix A)" }, "sequence_mixing": { "value": "SSM", "provenance": "derived", "note": "32/32 layers use the recurrent mLSTM cell (state-space/linear-recurrent family); no softmax-attention layers exist in the model" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true ('model weights, model code and training code are open-source'); training data fully described as public datasets (DCLM + public SFT sets) but no curated corpus is released by the authors" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "6.87B parameters (1–10B range)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "general-purpose 7B LLM (DCLM pre-training, standard leaderboard evals) whose primary design emphasis is inference speed and efficiency for test-time-compute workloads; edge-device/local deployment is cited as a target use case (§1, §5.2)" } }, "prose": { "design_overview": { "value": "xLSTM-7B is a 32-block, 6.87B-parameter fully recurrent LLM with no attention anywhere: every block pairs an output-gated multi-head mLSTM sequence-mix layer (8 heads, head dimension 512, embedding dimension 4096) with a SwiGLU channel-mix MLP (projection factor 2.66), both under a pre-norm RMSNorm. The mLSTM cell — inherited from Beck et al. (2024) — keeps a fixed-size matrix memory (C_t, n_t, m_t) per head: 134.2 MB total for the 8-head configuration, equivalent to a 128-token KV cache in a similar Transformer, with linear compute in sequence length and a fully parallel training mode via chunkwise kernels. The report's architectural contribution is the post-up projection block: running the mLSTM at the embedding dimension instead of an up-projected space, and dropping the channel-wise convolutions and learnable skip connections of the original xLSTM, which yields 2×–4× higher token throughput. Training stability at 7B scale comes from a four-part recipe: RMSNorm pre-norms, soft-capping of the input/forget gates at a=15, input-gate bias initialized to −10, and final-logit soft-capping at a=30 (Gemma-style). There is no positional encoding at all — order is carried implicitly by the recurrence — and no context extension: the model is trained and served at 8K, with a separate experimental LCTX variant reaching 32K via a long-context cool-down. Training ran on 128 H100s over 2.3T tokens (DCLM, then a math/coding/Q&A stage with public SFT datasets), using FSDP, batch ramp-up to 512, and EOD-token memory resets for sequence packing. Only the pre-trained base model is released.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "mLSTM cell with matrix memory (recurrent + parallel modes)", "provenance": "stated", "ref": "§2 ('we reassess the mLSTM (Beck et al., 2024), on which we build our xLSTM 7B')" }, "origin": { "value": "xLSTM (Beck et al., 2024)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "SwiGLU gated MLP as channel-mix layer (projection factor 2.66 'matching common Transformers')", "provenance": "stated", "ref": "§3.1, Fig. 8" }, "origin": { "value": "Transformer LLM practice", "provenance": "stated", "ref": "Fig. 8 caption" } }, { "mechanism": { "value": "RMSNorm pre-norm at block entry (replacing LayerNorm; practice from Transformer and Mamba training)", "provenance": "stated", "ref": "§3.2 (cites Zuo et al., 2024; Touvron et al., 2023; Gemma Team, 2024a)" }, "origin": { "value": "Transformer / Mamba training practice", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "Final-layer logit soft-capping with a=30 (a·tanh(x/a))", "provenance": "stated", "ref": "§3.2 ('similar to Gemma Team (2024b)')" }, "origin": { "value": "Gemma (Gemma Team, 2024b)", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "Chunkwise-parallel training kernels for the mLSTM cell", "provenance": "stated", "ref": "§2 (cites Anonymous, 2025)" }, "origin": { "value": "Anonymous (2025) mLSTM kernels (not in atlas)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "GPT-NeoX-20B byte-level BPE tokenizer (vocab 50,257)", "provenance": "stated", "ref": "Appendix A (cites Black et al., 2022)" }, "origin": { "value": "GPT-NeoX-20B", "provenance": "stated", "ref": "Appendix A" } } ], "influence_out": [], "notable_omissions": { "value": [ "sLSTM — the scalar-LSTM branch of the original xLSTM architecture (Beck et al., 2024) — is entirely absent; xLSTM-7B relies fully on mLSTM cells and the report never mentions sLSTM", "No positional encoding of any kind is discussed (order carried implicitly by recurrence)", "SwiGLU inner (FFN) dimension not printed; only the 2.66 projection factor is given", "Training precision (BF16/FP32/FP8) not disclosed", "No total compute (FLOPs/GPU-hours) or MFU disclosed", "No license named for the open-source release", "No quantisation formats, speculative decoding, or serving-system optimisations disclosed (speed benchmarks use torch.compile + CUDA graphs)", "No chat/instruction-tuned variant released — only the pre-trained base model", "Long-context capability is shown only for the experimental LCTX variant (Fig. 3); no long-context model is released", "RULER results reported only as figures, not tables of numbers" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "nxai-xlstm-7b.json" }, { "slug": { "value": "openai-gpt-2-xl-1-5b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/openai-gpt-2-xl-1-5b", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "GPT-2", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "117M", "provenance": "stated", "ref": "Table 2" }, { "value": "345M", "provenance": "stated", "ref": "Table 2" }, { "value": "762M", "provenance": "stated", "ref": "Table 2" }, { "value": "1542M", "provenance": "stated", "ref": "Table 2" } ], "org": { "value": "OpenAI", "provenance": "stated", "ref": "Title page byline / fn 1" }, "release_date": { "provenance": "unknown", "note": "Report is undated and not on arXiv (API title/author search returned nothing); no announcement dates used" }, "report_url": { "value": "https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "provenance": "unknown" }, "open_weights": { "provenance": "unknown" } }, "scale": { "total_params": { "value": 1.542, "provenance": "stated", "ref": "Table 2 (1542M)" }, "active_params_per_token": { "value": 1.542, "provenance": "derived", "note": "Dense model: active = total (1542M); report describes no MoE" }, "layers": { "value": 48, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 1600, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "provenance": "unknown" }, "ffn_ratio": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "head_dim": { "provenance": "unknown" }, "vocab_size": { "value": 50257, "provenance": "stated", "ref": "§2.2" }, "embedding_tied": { "provenance": "unknown" }, "variants": [ { "name": { "value": "117M", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 0.117, "provenance": "stated", "ref": "Table 2" }, "layers": { "value": 12, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 768, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 1024, "provenance": "inferred", "note": "Report describes a single context length of 1024 for the model (§2.3); no per-variant breakdown" } }, { "name": { "value": "345M", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 0.345, "provenance": "stated", "ref": "Table 2" }, "layers": { "value": 24, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 1024, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 1024, "provenance": "inferred", "note": "Report describes a single context length of 1024 for the model (§2.3); no per-variant breakdown" } }, { "name": { "value": "762M", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 0.762, "provenance": "stated", "ref": "Table 2" }, "layers": { "value": 36, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 1280, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 1024, "provenance": "inferred", "note": "Report describes a single context length of 1024 for the model (§2.3); no per-variant breakdown" } }, { "name": { "value": "1542M", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 1.542, "provenance": "stated", "ref": "Table 2" }, "layers": { "value": 48, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": 1600, "provenance": "stated", "ref": "Table 2" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "value": 1024, "provenance": "inferred", "note": "Report describes a single context length of 1024 for the model (§2.3); no per-variant breakdown" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "inferred", "note": "Absence rule: report describes the full block (attention + FFN, §2.3) with no routing/MoE anywhere" }, "moe": { "expert_count": { "provenance": "n/a", "ref": "§2.3 dense block" }, "experts_per_token": { "provenance": "n/a", "ref": "§2.3 dense block" }, "shared_experts": { "provenance": "n/a", "ref": "§2.3 dense block" }, "routing": { "provenance": "n/a", "ref": "§2.3 dense block" }, "load_balancing": { "provenance": "n/a", "ref": "§2.3 dense block" }, "expert_granularity": { "provenance": "n/a", "ref": "§2.3 dense block" } }, "attention_variant": { "provenance": "unknown" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "Absence rule: report describes standard stacked blocks with no per-layer pattern (§2.3)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard residual connections between sub-blocks described (§2.3)" }, "position_encoding": { "method": { "provenance": "unknown" }, "rope_base": { "provenance": "unknown" }, "partial_rope": { "provenance": "unknown" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "Report states context 1024 (§2.3) with no extension mechanism" }, "factor": { "provenance": "unknown" }, "target_length": { "provenance": "unknown" } } }, "normalization": { "type": { "value": "LayerNorm", "provenance": "stated", "ref": "§2.3 (Layer normalization moved to the input of each sub-block)" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§2.3 (Layer normalization moved to the input of each sub-block)" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: block fully described (§2.3), no QK-normalisation mentioned" } }, "activation": { "provenance": "unknown" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: full block description, no sink tokens" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule" }, "other": { "value": [ "residual-scaled initialization (1/√N)", "extra LayerNorm after the final self-attention block" ], "provenance": "stated", "ref": "§2.3" } } }, "context": { "trained_length": { "value": 1024, "provenance": "stated", "ref": "§2.3" }, "deployed_length": { "value": 1024, "provenance": "inferred", "note": "Report discloses a single context length (1024, §2.3); no separate deployed length" }, "extension_method": { "value": "none", "provenance": "inferred", "note": "Report trains at 1024 with no extension mechanism described" } }, "tokenizer": { "algorithm": { "value": "byte-level BPE", "provenance": "stated", "ref": "§2.2" }, "notes": { "value": "Byte-level base vocabulary of 256, no cross-category merges, space as exception (§2.2)", "provenance": "stated", "ref": "§2.2" } }, "training": { "tokens": { "provenance": "unknown" }, "data_composition": { "value": "WebText corpus: outbound links from Reddit with ≥3 karma (~45M links, ~8M documents, 40GB text); Wikipedia removed (§2.1)", "provenance": "stated", "ref": "§2.1" }, "curriculum": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "optimizer": { "provenance": "unknown" }, "lr_schedule": { "provenance": "unknown" }, "batch_schedule": { "value": "batch size 512", "provenance": "stated", "ref": "§2.3" }, "precision": { "provenance": "unknown" }, "parallelism": { "provenance": "unknown" }, "hardware": { "provenance": "unknown" }, "compute": { "provenance": "unknown" } }, "post_training": { "sft": { "value": false, "provenance": "inferred", "note": "Report describes no fine-tuning; model evaluated zero-shot (§3)" }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "reasoning_training": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "Report describes direct pretraining; no distillation" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes text tasks only; no non-text modality mentioned" }, "attachment": { "provenance": "n/a", "ref": "§2 model description" } }, "efficiency": { "kv_cache": { "value": "not discussed; report predates KV-cache analysis and discloses no attention grouping", "provenance": "unknown" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "evaluation": { "benchmarks": [ { "name": { "value": "LAMBADA (PPL)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 8.63, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LAMBADA (ACC)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 63.24, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "CBT-CN", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 93.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "CBT-NE", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.05, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "WikiText-2 (PPL)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 18.34, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "PTB (PPL)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 35.76, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "enwik8 (bpc)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 0.93, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "WikiText-103 (PPL)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 17.48, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "1BW (PPL)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 42.16, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "WMT-14 En-Fr (BLEU)", "provenance": "stated", "ref": "§3.7" }, "value": { "value": 11.5, "provenance": "stated", "ref": "§3.7" }, "ref": { "value": "§3.7", "provenance": "stated", "ref": "§3.7" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Language models as unsupervised multitask learners: zero-shot task transfer without task-specific supervision", "provenance": "stated", "ref": "Abstract / §1" }, "ref": { "value": "Abstract / §1", "provenance": "stated", "ref": "Abstract / §1" } }, { "text": { "value": "WebText corpus construction (outbound Reddit links, ≥3 karma)", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Byte-level BPE with no cross-category merges enabling universal encoding", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "Modified initialization, pre-norm placement, and 1024-token context improve zero-shot results", "provenance": "stated", "ref": "§2.3" }, "ref": { "value": "§2.3", "provenance": "stated", "ref": "§2.3" } } ], "assessment": { "value": "Pioneering record relative to the atlas: predates every other entry; established the decoder-only pretrain-then-zero-shot paradigm that the atlas's dense line (Llama 3.1, Phi-3) descends from; within-atlas novelty comparisons do not apply", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "from block: all-FFN-dense, no routing (absence rule)" }, "sequence_mixing": { "value": "full-attention", "provenance": "inferred", "note": "Report describes standard unrestricted attention over 1024-token context (§2.3); no restricted patterns" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "undisclosed", "provenance": "derived", "note": "report silent on weight release; only a footnote offers 'preliminary code' for the small model" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "1.542B total params (reference variant)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as general-purpose multitask learners (Abstract)" } }, "prose": { "design_overview": { "value": "GPT-2 is a decoder-only transformer stack with learned absolute positional embeddings, pre-norm LayerNorm placement, a byte-level BPE vocabulary, and a 1024-token context. The report's contribution is largely empirical: scaling a plain transformer to 1.5B parameters and showing zero-shot transfer across 8 of the 9 evaluated task families without task-specific fine-tuning. Architecturally it is the direct ancestor of the atlas's dense decoder line — same block shape that Llama 3.1, Phi-3 and OLMo 2 later refine with GQA, RoPE, SwiGLU and modern training recipes. Its historical interest in the atlas is as the baseline against which everything newer is measured: no KV-cache optimisation, no grouped attention, no positional-encoding extension story, and no disclosed compute or optimizer details.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "decoder-only transformer with pre-norm and residual connections", "provenance": "stated", "ref": "§2.3" }, "origin": { "value": "GPT (Radford et al. 2018)", "provenance": "stated", "ref": "§2.3" } } ], "influence_out": [ { "model_slug": { "value": "openai-gpt-oss", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Autoregressive MoE transformer architecture building upon the GPT-2 and GPT-3 architectures", "provenance": "inferred", "note": "mechanism as stated in openai-gpt-oss report's lineage_in" } } ], "notable_omissions": { "value": [ "No attention head counts, FFN inner dimensions, optimizer, learning rate, precision, hardware or compute disclosed", "Positional-encoding method never named (era-default learned absolute embeddings not stated)", "No KV-cache / serving discussion (report predates the concern)", "Weights release mentioned only as 'preliminary code' for the small model", "Report undated and not on arXiv — release_date unknown" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "openai-gpt-2-xl-1-5b.json" }, { "slug": { "value": "openai-gpt-oss", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/openai-gpt-oss", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "gpt-oss", "provenance": "stated", "ref": "Title ('gpt-oss-120b & gpt-oss-20b Model Card')" }, "variants": [ { "value": "20B", "provenance": "stated", "ref": "§2 ('We are releasing two model sizes: gpt-oss-120b ... and gpt-oss-20b')" }, { "value": "120B", "provenance": "stated", "ref": "§2 ('We are releasing two model sizes: gpt-oss-120b ... and gpt-oss-20b')" } ], "org": { "value": "OpenAI", "provenance": "stated", "ref": "Title page (OpenAI)" }, "release_date": { "value": "2025-08-05", "provenance": "stated", "ref": "Title page ('August 5, 2025')" }, "report_url": { "value": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0 (plus the gpt-oss usage policy)", "provenance": "stated", "ref": "§1 ('available under the Apache 2.0 license and our gpt-oss usage policy')" }, "open_weights": { "value": true, "provenance": "stated", "ref": "§1 ('two open-weight reasoning models available under the Apache 2.0 license')" } }, "scale": { "total_params": { "value": 116.83, "provenance": "stated", "ref": "Table 1", "note": "Table 1 prints 116.83B; §2 rounds to 116.8B. The card notes the '120b' name is nominal ('We refer to the models as “120b” and “20b” for simplicity, though they technically have 116.8B and 20.9B parameters')" }, "active_params_per_token": { "value": 5.13, "provenance": "stated", "ref": "Table 1", "note": "Table 1 prints 5.13B; §2 rounds to 5.1B ('active” parameters per token per forward pass'). Unembedding parameters count towards active, but not embeddings (Table 1 footnote)" }, "layers": { "value": 36, "provenance": "stated", "ref": "§2" }, "hidden_dim": { "value": 2880, "provenance": "stated", "ref": "§2.2 ('Both models have a residual stream dimension of 2880')" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "per-expert MoE inner dimension not disclosed (only the aggregate MLP parameter total, 114.71B, is given in Table 1)" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: expert inner dim not disclosed" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "§2.2 ('Each layer has 64 query heads of dimension 64')" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "§2.2 ('Grouped Query Attention (GQA) with 8 key-value heads')" }, "head_dim": { "value": 64, "provenance": "stated", "ref": "§2.2 ('64 query heads of dimension 64')" }, "vocab_size": { "value": 201088, "provenance": "stated", "ref": "§2.3 ('a total of 201,088 tokens')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "report counts 'Embed + Unembed' jointly (1.16B, Table 1) and notes unembedding parameters count toward active but embeddings do not; tying is not stated" }, "variants": [ { "name": { "value": "20B", "provenance": "stated", "ref": "§2" }, "total_params": { "value": 20.91, "provenance": "stated", "ref": "Table 1", "note": "Table 1 prints 20.91B; §2 rounds to 20.9B" }, "layers": { "value": 24, "provenance": "stated", "ref": "§2" }, "hidden_dim": { "value": 2880, "provenance": "stated", "ref": "§2.2 ('Both models have a residual stream dimension of 2880')" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "§2.2" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "§2.2" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§2.2", "note": "'we ... extend the context length of dense layers to 131,072 tokens using YaRN' (applies to both models)" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "§2 ('The gpt-oss models are autoregressive Mixture-of-Experts (MoE) transformers')", "note": "all MLP blocks are routed MoE; no dense layers disclosed" }, "moe": { "expert_count": { "value": 128, "provenance": "stated", "ref": "§2.2 ('a fixed number of experts (128 for gpt-oss-120b and 32 for gpt-oss-20b)')", "note": "128 for gpt-oss-120b (reference variant); 32 for gpt-oss-20b" }, "experts_per_token": { "value": 4, "provenance": "stated", "ref": "§2.2 ('we select the top-4 experts for each token')" }, "shared_experts": { "value": 0, "provenance": "inferred", "note": "MoE block fully described as 'a fixed number of experts ... as well as a standard linear router projection' (§2.2) with no shared/always-active expert mentioned" }, "routing": { "value": "standard linear router projection mapping residual activations to expert scores; top-4 selected per token; each expert's output weighted by the softmax of the router projection over only the selected experts", "provenance": "stated", "ref": "§2.2" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no load-balancing mechanism disclosed" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "expert width not disclosed; only expert counts (128/32) are given" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.2", "note": "GQA softmax attention (64 Q / 8 KV heads) whose layers alternate between banded-window (bandwidth 128) and fully dense patterns; not a single uniform pattern, so no single variant label fits" }, "attention_layer_pattern": { "value": "alternating banded-window (bandwidth 128) and fully dense attention layers, following GPT-3", "provenance": "stated", "ref": "§2.2" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: report describes the block in detail (§2.2) and never mentions attention over prior layer outputs" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.2 ('We apply rotary position embeddings')" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": false, "provenance": "inferred", "note": "absence rule: no partial-RoPE described" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "ref": "§2.2", "note": "'extend the context length of dense layers to 131,072 tokens using YaRN'" }, "factor": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§2.2" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.2 ('applying root mean square normalization [6] on the activations before each attention and MoE block')" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§2.2 ('Similar to GPT-2 we use Pre-LN placement')" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "absence rule: no QK-normalisation described; the learned softmax-denominator bias (§2.2) is a logit bias, not QK-norm" } }, "activation": { "value": "gated SwiGLU", "provenance": "stated", "ref": "§2.2", "note": "implementation is unconventional: 'Our SwiGLU implementation is unconventional, including clamping and a residual connection' (footnote 1)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "no designed sink tokens; the learned per-head softmax-denominator bias is an off-by-one-attention style mechanism that lets attention pay no attention to any tokens (§2.2) — related to the attention-sinks literature but not a sink-token design" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no attention-logit softcapping described" }, "other": [ { "value": "Learned per-head bias in the denominator of the softmax, similar to off-by-one attention and attention sinks, enabling the attention mechanism to pay no attention to any tokens", "provenance": "stated", "ref": "§2.2" }, { "value": "Unconventional gated SwiGLU implementation including clamping and a residual connection", "provenance": "stated", "ref": "§2.2 (footnote 1)" } ] } }, "context": { "trained_length": { "value": null, "provenance": "unknown", "note": "pre-training sequence length not disclosed; only the YaRN-extended dense-layer length of 131,072 is given" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§2.2" }, "extension_method": { "value": "YaRN", "provenance": "stated", "ref": "§2.2", "note": "applied to the dense attention layers" } }, "tokenizer": { "algorithm": { "value": "Byte Pair Encoding (BPE)", "provenance": "stated", "ref": "§2.3" }, "notes": { "value": "o200k_harmony tokenizer: extends the o200k tokenizer used for other OpenAI models such as GPT-4o and o4-mini with tokens explicitly used for the harmony chat format; open-sourced in the TikToken library; 201,088 total tokens (vocab size lives in scale.vocab_size)", "provenance": "stated", "ref": "§2.3" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "only 'trillions of tokens' disclosed (§2.4); no exact count" }, "data_composition": { "value": "text-only dataset with trillions of tokens, with a focus on STEM, coding, and general knowledge; data filtered for harmful content in pre-training, especially hazardous biosecurity knowledge, by reusing the CBRN pre-training filters from GPT-4o; knowledge cutoff of June 2024", "provenance": "stated", "ref": "§2.4" }, "curriculum": { "value": "none disclosed", "provenance": "inferred", "note": "no staged training, annealing, or context-length curriculum described" }, "optimizer": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "master training precision not stated; post-training quantization of MoE weights to MXFP4 (4.25 bits per parameter) is described in §2.1" }, "parallelism": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hardware": { "value": "NVIDIA H100 GPUs; PyTorch framework with expert-optimized Triton kernels; Flash Attention used to reduce memory and accelerate training", "provenance": "stated", "ref": "§2.4" }, "compute": { "value": "2.1 million H100-hours for gpt-oss-120b; gpt-oss-20b 'needing almost 10x fewer'", "provenance": "stated", "ref": "§2.4" } }, "post_training": { "sft": { "value": true, "provenance": "inferred", "note": "instruction-hierarchy training is described as supervised ('we ... supervised gpt-oss to follow the instructions in the system message over developer messages', §4.3); the main post-training is described as o3-style CoT RL (§2.5); no general SFT stage is named" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§2.5", "note": "'we post-train the models using similar CoT RL techniques as OpenAI o3'; deliberative alignment is used to teach refusals and jailbreak robustness (§3). The specific RL algorithm (PPO/GRPO/etc.) is never named" }, "reasoning_training": { "value": "o3-style CoT RL teaching reasoning and tool use; variable-effort reasoning training with three levels (low, medium, high) configured via system-prompt keywords such as 'Reasoning: low'; agentic tool use trained via a browsing tool (search/open functions), a Python tool in a stateful Jupyter notebook, and arbitrary developer functions; deliberative alignment for refusal behavior; instruction hierarchy training (System > Developer > User > Assistant > Tool)", "provenance": "stated", "ref": "§2.5, §3" }, "distillation": { "value": "none disclosed", "provenance": "inferred", "note": "no distillation from a teacher model mentioned" } }, "modality": { "type": { "value": "text-only", "provenance": "stated", "ref": "§1 ('these text-only models')" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§1", "note": "text-only; no non-text modality" } }, "efficiency": { "kv_cache": { "value": "GQA with 8 KV heads: KV cache scales with 8 shared heads instead of 64 query heads; banded-window layers restrict attention (and cache use) to a 128-token band", "provenance": "derived", "note": "consequence of attention_variant (GQA, 64Q/8KV, §2.2); the report itself does not discuss KV-cache sizing" }, "quantization": { "value": "MXFP4 for MoE weights (4.25 bits per parameter), applied via post-training quantization; enables gpt-oss-120b to fit on a single 80GB GPU and gpt-oss-20b on systems with as little as 16GB memory; checkpoint sizes 60.8GiB / 12.8GiB", "provenance": "stated", "ref": "§2.1, Table 1" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding, draft models, or multi-token prediction mentioned" }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "no serving optimisations disclosed; the models are stated to be compatible with OpenAI's Responses API (§1)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "AIME 2024 (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 95.8, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME 2024 (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 96.6, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME 2025 (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 92.5, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME 2025 (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 97.9, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA Diamond (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 80.1, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA Diamond (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 80.9, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (no tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 14.9, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (with tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 19.0, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 90.0, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Bench Verified", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 62.4, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Tau-Bench Retail", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 67.8, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Tau-Bench Airline", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 49.2, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Aider Polyglot", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 44.4, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMMLU (Average)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 81.3, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HealthBench", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 57.6, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HealthBench Hard", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 30.0, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HealthBench Consensus", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 89.9, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Codeforces (no tools, Elo)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 2463, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Codeforces (with tools, Elo)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 2622, "provenance": "stated", "ref": "Table 3 (gpt-oss-120b, high reasoning level)" }, "ref": { "value": "Table 3 (gpt-oss-120b, high reasoning level)", "provenance": "stated", "ref": "Table 3" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Introduces gpt-oss-120b and gpt-oss-20b, two open-weight reasoning models under the Apache 2.0 license with full chain-of-thought, Structured Outputs, and adjustable reasoning effort", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } }, { "text": { "value": "MoE with top-4 routing: 128 experts (120b) / 32 experts (20b), gated SwiGLU, and softmax weighting over only the selected experts", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "GPT-3-style alternating banded-window (bandwidth 128) and fully dense attention with GQA (64 Q / 8 KV heads, head dim 64), RoPE, and a learned per-head softmax-denominator bias (off-by-one attention / attention sinks) enabling attention to pay no attention to any tokens; dense layers extended to 131,072 tokens via YaRN", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "MXFP4 post-training quantization of MoE weights (4.25 bits per parameter) fitting gpt-oss-120b on a single 80GB GPU and gpt-oss-20b in ~16GB memory", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "o200k_harmony tokenizer open-sourced in the TikToken library, extending the o200k tokenizer with harmony chat format tokens (201,088 total)", "provenance": "stated", "ref": "§2.3" }, "ref": { "value": "§2.3", "provenance": "stated", "ref": "§2.3" } }, { "text": { "value": "Harmony chat format with role-based instruction hierarchy (System > Developer > User > Assistant > Tool) and channels marking CoT, commentary, and final-answer visibility for agentic workflows", "provenance": "stated", "ref": "§2.5.1" }, "ref": { "value": "§2.5.1", "provenance": "stated", "ref": "§2.5.1" } }, { "text": { "value": "Safety: Preparedness Framework evaluations including adversarially fine-tuned versions of gpt-oss-120b (SAG-reviewed), concluding the model does not reach High capability in Biological/Chemical or Cyber risk", "provenance": "stated", "ref": "§1, §5" }, "ref": { "value": "§1, §5", "provenance": "stated", "ref": "§1, §5" } } ], "assessment": { "value": "A conventional decoder-only MoE in the GPT-2/GPT-3 lineage with two distinctive touches: the revived GPT-3 sparse-transformer pattern (alternating banded-window/dense attention, here with GQA, RoPE and YaRN extension) and a learned per-head softmax-denominator bias for 'attend to nothing' (off-by-one attention family). vs DeepSeek-V3, the closest atlas MoE peer: far shallower (36 vs 61 layers), far fewer active params (5.1B vs 37B), top-4 of 128 experts with no shared expert, no MLA, no disclosed load balancing, and no MTP; the report also lacks any MoE training-stability discussion. The MXFP4-native release — quantization applied during post-training at 4.25 bits/parameter and treated as a first-class artifact (checkpoint sizes given in GiB) — is distinctive among atlas entries. The reasoning post-training (o3-style CoT RL with variable low/medium/high effort) extends the deepseek-r1 RLVR line with an explicit test-time effort control that no other atlas record implements; the model card's safety apparatus (adversarial fine-tuning simulations) is unique in the atlas but not architecture.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, deepseek-r1, mistral-mixtral-8x7b, moonshot-kimi-k3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "all MLP blocks are routed MoE (128/32 experts, top-4, §2.2); block_type sparse-MoE" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "attention layers alternate banded-window and fully dense patterns (~50/50 per §2.2); neither pattern reaches the 80% majority threshold" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (stated in §1)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Apache 2.0, §1); data composition described qualitatively but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "116.83B ≥ 100B (reference variant gpt-oss-120b)" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "framed as 'two open-weight reasoning models' with o3-style CoT RL, variable reasoning effort (low/medium/high), and agentic tool use (§1, §2.5); text-only, so not a multimodal frontier generalist" } }, "prose": { "design_overview": { "value": "gpt-oss-120b and gpt-oss-20b are OpenAI's first open-weight MoE release: 36-layer (24-layer) decoder-only transformers with a 2880-wide residual stream, RMSNorm with Pre-LN placement, and an explicit GPT-2/GPT-3 lineage. Every MLP block is a router MoE — 128 experts (120b) or 32 (20b) with a linear router selecting the top-4 experts per token, outputs weighted by a softmax over the selected experts only, using a gated SwiGLU activation that (unusually) includes clamping and an internal residual connection. Attention follows the GPT-3 sparse-transformer pattern, alternating banded-window (128-token bandwidth) and fully dense layers; each layer has 64 query heads of dimension 64 with GQA over 8 KV heads, RoPE, and a learned per-head bias in the softmax denominator (off-by-one attention / attention-sinks style) that lets a head attend to nothing. Dense layers are extended to 131,072 tokens via YaRN. The card's distinctive engineering is deployment-focused: MoE weights are quantized post-training to MXFP4 (4.25 bits/parameter), fitting 120b on a single 80GB GPU and 20b in ~16GB, and the o200k_harmony tokenizer adds chat-format tokens. Post-training is o3-style CoT RL with three configurable reasoning levels, a harmony chat format with a System > Developer > User > Assistant > Tool hierarchy, and agentic tool use (browse, Python, developer functions). As a model card, it discloses far less architecture than technical reports of comparable MoE peers.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Autoregressive MoE transformer architecture building upon the GPT-2 and GPT-3 architectures", "provenance": "stated", "ref": "§2" }, "origin": { "value": "GPT-2 / GPT-3", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Alternating banded-window and fully dense attention layers", "provenance": "stated", "ref": "§2.2 (cites Sparse Transformer [10], GPT-3 [11])" }, "origin": { "value": "GPT-3 / Sparse Transformer", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Grouped Query Attention (GQA, 64 Q / 8 KV heads)", "provenance": "stated", "ref": "§2.2 (cites GQA [12], MQA [13])" }, "origin": { "value": "GQA (Ainslie et al. 2023)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Rotary position embeddings (RoPE)", "provenance": "stated", "ref": "§2.2 (cites RoFormer [14])" }, "origin": { "value": "RoFormer (Su et al.)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "YaRN context extension to 131,072 tokens", "provenance": "stated", "ref": "§2.2 (cites YaRN [15])" }, "origin": { "value": "YaRN (Peng et al. 2023)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Learned per-head bias in the softmax denominator (off-by-one attention / attention sinks)", "provenance": "stated", "ref": "§2.2 (cites [16] off-by-one, [17] attention sinks)" }, "origin": { "value": "Off-by-one attention (Miller 2023) / attention sinks (Xiao et al. 2023)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "CoT reinforcement-learning post-training techniques", "provenance": "stated", "ref": "§2.5 ('using similar CoT RL techniques as OpenAI o3')" }, "origin": { "value": "OpenAI o3", "provenance": "stated", "ref": "§2.5" } }, { "mechanism": { "value": "o200k tokenizer base extended with harmony chat format tokens (o200k_harmony)", "provenance": "stated", "ref": "§2.3" }, "origin": { "value": "o200k (GPT-4o / o4-mini)", "provenance": "stated", "ref": "§2.3" } } ], "influence_out": [ { "model_slug": { "value": "xiaomi-mimo-v2-flash-309b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Learnable attention sink bias in the softmax denominator", "provenance": "inferred", "note": "mechanism as stated in xiaomi-mimo-v2-flash-309b report's lineage_in" } } ], "notable_omissions": { "value": [ "No pre-training token count (only 'trillions of tokens', §2.4)", "No pre-training sequence length; only the YaRN-extended 131,072-token dense-layer context is given", "Per-expert MoE inner dimension not disclosed (ffn_inner_dim / ffn_ratio unknown)", "No MoE load-balancing mechanism disclosed", "No RoPE base frequency, partial-RoPE, or YaRN factor disclosed", "Master training precision, optimizer, LR and batch schedules, and parallelism not disclosed", "Input/output embedding tying not stated (only a joint 'Embed + Unembed' 1.16B count, Table 1)", "RL algorithm beyond 'CoT RL similar to o3' never named", "No data mixture percentages and no full license text for the gpt-oss usage policy", "No architectural ablations or scaling-law studies (safety-focused model card)", "Benchmark comparisons limited to OpenAI's own o3 / o3-mini / o4-mini family; no independent third-party leaderboards" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "openai-gpt-oss.json" }, { "slug": { "value": "poolside-laguna-xs-2-33b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://poolside.ai/assets/laguna/laguna-m1-xs2-technical-report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/poolside-laguna-xs-2-33b", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Laguna", "provenance": "stated", "ref": "Title (Laguna M1 XS2 technical report)" }, "variants": [ { "value": "XS.2 (33.4B)", "provenance": "stated", "ref": "config tables" }, { "value": "M.1 (225.8B)", "provenance": "stated", "ref": "config tables" } ], "org": { "value": "poolside", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "provenance": "unknown", "note": "Undated non-arXiv PDF; no announcement dates used" }, "report_url": { "value": "https://poolside.ai/assets/laguna/laguna-m1-xs2-technical-report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "license section (stated twice)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "license/release section" } }, "scale": { "total_params": { "value": 33.4, "provenance": "stated", "ref": "config tables (XS.2)" }, "active_params_per_token": { "value": 3, "provenance": "stated", "ref": "config tables (XS.2 3B active)" }, "layers": { "value": 40, "provenance": "stated", "ref": "quantization section" }, "hidden_dim": { "provenance": "unknown", "note": "Not disclosed for XS.2; only ablation-proxy table values (2048)" }, "ffn_inner_dim": { "provenance": "unknown", "note": "Not disclosed for XS.2" }, "ffn_ratio": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown", "note": "Per-layer-type allocation: 48 Q-heads on GA layers, 64 Q-heads on SWA layers — no single model-level value" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "config tables (GQA 8 KV heads)" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "config tables" }, "vocab_size": { "value": 100352, "provenance": "stated", "ref": "tokenizer section" }, "embedding_tied": { "provenance": "unknown" }, "variants": [ { "name": { "value": "XS.2 (33.4B)", "provenance": "stated", "ref": "config tables" }, "total_params": { "value": 33.4, "provenance": "stated", "ref": "config tables" }, "layers": { "value": 40, "provenance": "stated", "ref": "quantization section" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "config tables" }, "context_length": { "value": 262144, "provenance": "stated", "ref": "context section (256K deployed)" } }, { "name": { "value": "M.1 (225.8B)", "provenance": "stated", "ref": "config tables" }, "total_params": { "value": 225.8, "provenance": "stated", "ref": "config tables" }, "layers": { "provenance": "unknown" }, "hidden_dim": { "provenance": "unknown" }, "ffn_inner_dim": { "provenance": "unknown" }, "attention_heads_q": { "provenance": "unknown" }, "attention_heads_kv": { "provenance": "unknown" }, "context_length": { "provenance": "unknown" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "39 of 40 layers routed MoE (first layer dense)" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "config tables" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "config tables" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "config tables" }, "routing": { "value": "Sigmoid router with post-top-k score normalization; routed-expert modulation (×2.5)", "provenance": "stated", "ref": "architecture section" }, "load_balancing": { "value": "Aux loss (Qiu et al.) load balancing", "provenance": "stated", "ref": "architecture section" }, "expert_granularity": { "provenance": "unknown" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "note": "Interleaved sliding-window / global attention with per-head gating", "ref": "architecture section" }, "attention_layer_pattern": { "value": "3:1 SWA:GA interleaving (75% SWA); SWA layers window 512, 64 Q-heads, RoPE base 10k; GA layers 48 Q-heads, RoPE base 500k, partial RoPE 50%; per-head softplus gating; distinct Q-head counts and theta per attention type (layer-wise attention budgeting)", "provenance": "stated", "ref": "architecture section" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "architecture section (two bases, per attention type)" }, "rope_base": { "provenance": "unknown", "note": "Two bases disclosed: 10k (SWA) and 500k (GA) — no single model-level value" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "architecture section (50% of dims on GA layers)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "YaRN on GA layers to 128K, then RoPE base doubling to 256K without training", "ref": "context section" }, "factor": { "provenance": "unknown" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "context section" } } }, "normalization": { "type": { "provenance": "unknown" }, "placement": { "provenance": "unknown" }, "qk_norm": { "provenance": "unknown" } }, "activation": { "provenance": "unknown" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: detailed architecture description, no sink tokens" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule" }, "other": { "value": [], "provenance": "inferred", "note": "No stability tricks disclosed" } } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "context section (YaRN extension trained to 128K)" }, "deployed_length": { "value": 262144, "provenance": "stated", "ref": "context section (RoPE-scale doubling to 256K without training)" }, "extension_method": { "value": "YaRN on GA layers to 128K; RoPE base doubling to 256K (no training)", "provenance": "stated", "ref": "context section" } }, "tokenizer": { "algorithm": { "value": "BPE", "provenance": "stated", "ref": "tokenizer section" }, "notes": { "value": "100,352-token vocabulary", "provenance": "stated", "ref": "tokenizer section" } }, "training": { "tokens": { "value": 30, "provenance": "stated", "ref": "training section (>30T pre-training tokens)" }, "data_composition": { "provenance": "unknown" }, "curriculum": { "value": "none disclosed (pretraining); mid-train ~60B + SFT 3×40B + RL in post-training", "provenance": "stated", "ref": "training section" }, "optimizer": { "value": "Muon (Moonlight variant)", "provenance": "stated", "ref": "training section" }, "lr_schedule": { "value": "WSD schedule, peak LR 5e-4", "provenance": "stated", "ref": "training section" }, "batch_schedule": { "provenance": "unknown" }, "precision": { "value": "BF16 with FP32 master weights", "provenance": "stated", "ref": "training section" }, "parallelism": { "provenance": "unknown" }, "hardware": { "value": "2,048 H200 GPUs (M.1: 6,144)", "provenance": "stated", "ref": "training section" }, "compute": { "provenance": "unknown" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "post-training section (SFT, 3×40B)" }, "preference_optimization": { "value": "other", "provenance": "stated", "note": "CISPO: RL with verifiable rewards only, length-weighted leave-one-out advantage", "ref": "post-training section" }, "reasoning_training": { "value": "RL with verifiable rewards (CISPO); agentic coding emphasis", "provenance": "stated", "ref": "post-training section" }, "distillation": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Report describes code/text tasks only" }, "attachment": { "provenance": "n/a", "ref": "model description" } }, "efficiency": { "kv_cache": { "value": "SWA (window 512) on 75% of layers shrinks cache vs full attention; GA layers keep partial RoPE with 48 Q-heads; no latent-KV compression disclosed", "provenance": "derived", "note": "from attention pattern" }, "quantization": { "value": "QAD (quantization-aware distillation) and SpinQuant", "provenance": "stated", "ref": "quantization section" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "pipeline" } }, "evaluation": { "benchmarks": [ { "name": { "value": "SWE-bench Verified", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 69.9, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "SWE-bench Multilingual", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 57.7, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "SWE-Bench Pro", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 46.3, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "Terminal-Bench 2.0", "provenance": "stated", "ref": "eval tables" }, "value": { "value": 35.7, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "BBH", "provenance": "stated", "ref": "eval tables (base)" }, "value": { "value": 80.9, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "MMLU-STEM", "provenance": "stated", "ref": "eval tables (base)" }, "value": { "value": 78.1, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "GSM8K", "provenance": "stated", "ref": "eval tables (base)" }, "value": { "value": 84.2, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } }, { "name": { "value": "LiveCodeBench v6", "provenance": "stated", "ref": "eval tables (base)" }, "value": { "value": 29.3, "provenance": "stated", "ref": "eval tables" }, "ref": { "value": "eval tables", "provenance": "stated", "ref": "eval tables" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Layer-wise attention budgeting: per-layer attention-type allocation (3:1 SWA:GA) with per-head softplus gating and per-type Q-head counts and RoPE bases", "provenance": "stated", "ref": "architecture section" }, "ref": { "value": "architecture section", "provenance": "stated", "ref": "architecture section" } }, { "text": { "value": "CISPO reinforcement learning with verifiable rewards only", "provenance": "stated", "ref": "post-training section" }, "ref": { "value": "post-training section", "provenance": "stated", "ref": "post-training section" } }, { "text": { "value": "Long-horizon agentic coding focus with 256K deployed context", "provenance": "stated", "ref": "Abstract / context section" }, "ref": { "value": "Abstract / context section", "provenance": "stated", "ref": "Abstract / context section" } } ], "assessment": { "value": "The first per-head attention-budget design in the atlas: the SWA/GA interleave itself is familiar (Gemma 3, OLMo 3, Trinity), but allocating distinct Q-head counts and RoPE bases per attention type is a novel attention-budgeting pattern. The DeepSeekMoE-style shared+routed structure with routed-expert modulation is inherited (stated 'similar to DeepSeek-V3').", "provenance": "inferred", "note": "authored by atlas analyst" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "39/40 layers routed MoE" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "75% SWA / 25% GA — below the 80% threshold; deliberate 3:1 interleave" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Apache 2.0); data not released" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "33.4B reference variant" }, "design_intent": { "value": "domain-specific", "provenance": "inferred", "note": "framed for long-horizon agentic coding (Abstract)" } }, "prose": { "design_overview": { "value": "Laguna XS.2 is poolside's agentic-coding MoE: 33.4B total / 3B active over 40 layers, with 256 routed experts (top-8) plus one shared expert and a sigmoid router with post-top-k score normalization. Its distinguishing feature is layer-wise attention budgeting: a 3:1 interleave of sliding-window (window 512, 64 Q-heads, RoPE base 10k) and global (48 Q-heads, RoPE base 500k, partial RoPE 50%) attention, with per-head softplus gating deciding each head's contribution. Context grows from 4K pretraining through 32K and a YaRN stage to 128K, then doubles to 256K at deployment via RoPE-base scaling without further training. Training uses Muon (Moonlight variant) under WSD with >30T tokens on 2,048 H200s; post-training is SFT (3×40B) plus CISPO, an RL stage restricted to verifiable rewards with length-weighted leave-one-out advantages. The sibling M.1 (225.8B) shares the block at larger scale.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Shared + routed experts with routed-expert modulation ('similar to DeepSeek-V3')", "provenance": "stated", "ref": "architecture section" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "architecture section" } }, { "mechanism": { "value": "Muon optimizer (Moonlight variant)", "provenance": "stated", "ref": "training section" }, "origin": { "value": "Moonlight", "provenance": "stated", "ref": "training section" } } ], "influence_out": [], "notable_omissions": { "value": [ "Hidden dimension and FFN inner dimension not disclosed for XS.2", "No model-level Q-head count (per-layer-type allocation only)", "No normalization type or activation function disclosed", "No expert granularity, no training parallelism, no compute (FLOPs)", "No tokenizer algorithm details beyond BPE + 100,352 vocab" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "poolside-laguna-xs-2-33b.json" }, { "slug": { "value": "prime-intellect-intellect-3", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://storage.googleapis.com/intellect-3-paper/INTELLECT_3_Technical_Report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/prime-intellect-intellect-3/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "INTELLECT-3", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "Prime Intellect (Prime Intellect, Inc.)", "provenance": "stated", "ref": "Title page (Prime Intellect Team; correspondence to johannes@primeintellect.ai)" }, "release_date": { "value": null, "provenance": "unknown", "note": "PDF prints no date; not an arXiv submission, so no arXiv v1 metadata applies; reference [23] is accessed 25 Nov 2025 and the report mentions Qwen3.5-4B-style 2026 context only in the Tiny Aya comparison — no publication date can be established from the report" }, "report_url": { "value": "https://storage.googleapis.com/intellect-3-paper/INTELLECT_3_Technical_Report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named in the report (release link: huggingface.co/PrimeIntellect/INTELLECT-3)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('We open-source the model together with the full infrastructure stack'); §1 (footnote 2: huggingface.co/PrimeIntellect/INTELLECT-3); §5 ('By open sourcing INTELLECT-3, the environments, and the complete training framework')" } }, "scale": { "total_params": { "value": 106, "provenance": "stated", "ref": "Abstract ('a 106B-parameter Mixture-of-Experts model (12B active)')" }, "active_params_per_token": { "value": 12, "provenance": "stated", "ref": "Abstract ('12B active')" }, "layers": { "value": 46, "provenance": "stated", "ref": "§2.1.6 ('the activation memory for a sequence length of 48k, hidden size 4096, and 46 decoder layers is Memact = 46 × (48,000 × 4,096) × 2 bytes ≈ 18 GB')" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "§2.1.6 ('hidden size 4096'), Figure 5 caption ('hidden dim 4096')" }, "ffn_inner_dim": { "value": 1408, "provenance": "stated", "ref": "Figure 5 caption ('Execution time and TFLOPS of torch._grouped_mm with hidden dim 4096 and MoE dim 1408 on H200 SXM')", "note": "MoE per-expert intermediate dim as used in the training configuration; the report does not otherwise tabulate the config" }, "ffn_ratio": { "value": 0.34, "provenance": "derived", "note": "per-expert MoE ratio: 1408 / 4096 = 0.34375 (Figure 5 caption values)" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "attention configuration not disclosed (inherited from the GLM-4.5-Air base model, not restated)" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed (inherited from GLM-4.5-Air)" }, "head_dim": { "value": null, "provenance": "unknown", "note": "not disclosed (hidden 4096 / heads not computable without the head count)" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "report states the model is a Mixture-of-Experts model with 106B total / 12B active (Abstract, §2.1); the per-layer dense/MoE split is not disclosed (inherited from GLM-4.5-Air)" }, "moe": { "expert_count": { "value": null, "provenance": "unknown", "note": "not disclosed; Figure 5's 'Number of Experts (E)' sweep (up to 512) is a torch._grouped_mm kernel benchmark, not the model's expert count" }, "experts_per_token": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "routing": { "value": null, "provenance": "unknown", "note": "not disclosed (MoE layer implementation taken from torchtitan, §2.1.8)" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no balancing method stated; the MaxViolation metric is only logged as a diagnostic (cited to the auxiliary-loss-free balancing paper [47], §2.1.8)" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "not disclosed" } }, "attention_variant": { "value": null, "provenance": "unknown", "note": "attention internals not described (inherited from GLM-4.5-Air base); training uses Flash Attention 3 and standard attention-matrix analysis (§2.1.6)" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: no attention over prior layer outputs described; full activation checkpointing stores 'the outputs of each decoder layer' (§2.1.6), implying standard sequential residual layers" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "not described (inherited from GLM-4.5-Air)" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "no extension mechanism described (RL trains at up to 65,536 context, §3.3; agentic SFT at 98K via context parallelism, §3.2)" }, "factor": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "target_length": { "value": null, "provenance": "unknown", "note": "no extension target disclosed" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "not described (inherited from GLM-4.5-Air)" }, "placement": { "value": null, "provenance": "unknown", "note": "not described" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "report does not describe attention internals at all (inherited base model), so the absence rule for fully-described architectures does not apply" } }, "activation": { "value": null, "provenance": "unknown", "note": "not described (inherited from GLM-4.5-Air)" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "block internals not described in this report (inherited from GLM-4.5-Air); no sink mechanism mentioned" }, "softcapping": { "value": null, "provenance": "unknown", "note": "not described" }, "other": { "value": null, "provenance": "unknown", "note": "no stability mechanisms discussed at the architecture level" } } }, "context": { "trained_length": { "value": 65536, "provenance": "stated", "ref": "§3.3 ('a maximum context length of 65536' during RL), §3.2 (general SFT at 65K context; agentic SFT scaled to 98K context via context parallelism), §2.1.6 (sequence lengths up to 72k with activation offloading)", "note": "the RL stage — the main training stage — runs at 65,536; the agentic SFT stage reached 98K; the base model's native context is not restated" }, "deployed_length": { "value": null, "provenance": "unknown", "note": "deployment context length not stated in the report" }, "extension_method": { "value": null, "provenance": "unknown", "note": "no positional-encoding extension described; long-context training is achieved via context parallelism and activation offloading (§2.1.6, §3.2)" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "not described (inherited from GLM-4.5-Air)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "no pretraining in this report: INTELLECT-3 is a post-train of the GLM-4.5-Air base model (§1, §3), so no pretraining token count applies" }, "data_composition": { "value": "SFT (Table 1): OpenReasoning-Math (2M/78.1B), OpenReasoning-Code (1.9M/94.3B), OpenReasoning-Science (310K/32B), OpenReasoning-Tool (800K/3.8B), AM General Chat (952K/8.4B), AM Instruction Following (54K/400M), SWE Swiss (10.3K/700M), Toucan Tool (116K/700M), Environments Mix (38.4K/1.9B); main sources are NVIDIA's Nemotron-Post-Training-Dataset-v1 and AM-DeepSeek-R1-0528-Distilled (synthetic reasoning traces from DeepSeek-R1-0528). RL environments: math (21.2K problems from Skywork-OR1, Acereason-Math, DAPO, ORZ-Hard), code (8.6K Python problems, DeepCoder-inspired, SYNTHETIC-2), science (29.3K from MegaScience), logic (11.6K from SynLogic), deep research (DeepDive: 1K SFT + 2.2K RL samples, Serper web search), software engineering (R2E-Gym and mini-swe-agent-plus scaffolds)", "provenance": "stated", "ref": "§3.1, Table 1, §3.2" }, "curriculum": { "value": "two-stage SFT (general chat-and-reasoning at 65K context, then agentic SFT at 98K context) followed by RL with online difficulty filtering (problems sorted into easy/normal/hard pools by observed solve rate, trivial rollouts discarded, easy-pool prompts with pass rate 1 removed); max_off_policy_steps = 8", "provenance": "stated", "ref": "§3.2, §3.3, §2.1.5" }, "optimizer": { "value": "Muon (distributed via an all-to-all-based shard reshuffle for FSDP; Newton-Schulz update computed per rank on gathered gradient shards); weight decay 0.01 in SFT stage 1", "provenance": "stated", "ref": "§3.2, §2.1.7" }, "lr_schedule": { "value": "SFT stage 1: linear warmup from 1e-8 to 5e-5 over 300 steps; SFT stage 2: 5e-8 with linear decay over 800 steps; RL: Muon learning rate 1e-6", "provenance": "stated", "ref": "§3.2, §3.3" }, "batch_schedule": { "value": "SFT: ~33M tokens per step, FSDP world size 64 with DP replicate size 8 across 512 GPUs; RL: batch of 256 prompts with 16 rollouts per prompt", "provenance": "stated", "ref": "§3.2, §3.3" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not stated (Flash Attention 3 and FSDP2 used, §2.1.6, §2.1.1; reference [38] on FP16 mismatch is cited but not applied as a stated choice)" }, "parallelism": { "value": "FSDP2 trainer (with tensor/context/expert parallelism support in torchtitan; expert parallelism evaluated and NOT enabled because grouped-GEMM kernels already saturate at the training's sequence length/hidden dim); vLLM inference backend; context parallelism used to reach 98K in agentic SFT; FSDP degree 32 at 48K sequence length; activation offloading to CPU for long sequences", "provenance": "stated", "ref": "§2.1.1, §2.1.6, §2.1.8, §3.2" }, "hardware": { "value": "512 NVIDIA H200 GPUs across 64 nodes (400Gbps NDR InfiniBand, ConnectX-7; Lustre + NVMe-NFS storage; Slurm with Cgroup v2); RL used 60 nodes (16 training / 44 inference at ~1:3)", "provenance": "stated", "ref": "§2.4, §3.3" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed (only 'two months' on the 512-H200 cluster, §3)" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.2 (general chat-and-reasoning SFT stage plus agentic SFT stage; chat template with <|think|> token and Qwen3/GLM-style control tokens; always-reasoning model)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§3.3 (RLVR objective J_IcePop: masked token-level importance sampling with double-sided masking M(k) for k∈[α,β], α=0.5, β=5, group-relative advantages Âi,t = Si − mean({Si}G); early ablations compared GSPO vs CISPO)", "note": "policy-gradient RL with verifiable rewards; not named GRPO/PPO/RLHF by the report" }, "reasoning_training": { "value": "large-scale RLVR across math, code, science, logic, deep-research (web search) and software-engineering environments with verifiable rewards (rule-based math-verify plus CompassVerifier-7B LLM-judge; sandbox test execution; binary search rewards); long chain-of-thought with an always-reasoning chat template (<|think|> token, reasoning retained across turns); multi-turn agentic tool use (up to 200 turns in SWE scaffolds); online difficulty filtering curriculum", "provenance": "stated", "ref": "§3.1, §3.2, §3.3" }, "distillation": { "value": "none", "provenance": "inferred", "note": "post-train of the GLM-4.5-Air base model (§1, §3), not a distilled student; DeepSeek-R1-0528-generated SFT traces (Nemotron-PTD-v1, AM-DeepSeek-R1-0528-Distilled) are teacher-generated training data, which per schema is not distillation — noted in prose" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "no non-text modality described; all environments and benchmarks (math, code, science, logic, search, SWE, AIME, LCB, GPQA, HLE, MMLU-Pro) are text/tool-based" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1, §4 (text-only model)" } }, "efficiency": { "kv_cache": { "value": null, "provenance": "unknown", "note": "KV-cache design not discussed (attention internals inherited from GLM-4.5-Air and not restated)" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "§A.2 (the report only notes that evaluations avoid quantization loss by using creator APIs where available)" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "§2 (no speculative decoding mentioned)" }, "serving": { "value": "vLLM OpenAI-compatible inference backend used for rollout generation, with custom /update_weights and /reload_weights endpoints for policy updates (§2.1.1); no other serving optimizations disclosed", "provenance": "stated", "ref": "§2.1.1" } }, "evaluation": { "benchmarks": [ { "name": { "value": "AIME 2024", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 90.8, "provenance": "stated", "ref": "Table 2 (Avg@32)" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AIME 2025", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 88.0, "provenance": "stated", "ref": "Table 2 (Avg@32)" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "LiveCodeBench v6", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 69.3, "provenance": "stated", "ref": "Table 2 (Avg@2, 454 problems)" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA (Diamond)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 74.4, "provenance": "stated", "ref": "Table 2 (Avg@4, 198 questions)" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HLE", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 14.6, "provenance": "stated", "ref": "Table 2 (text-only subset, 2,158 examples, no tools)" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 81.9, "provenance": "stated", "ref": "Table 2 (Avg@1, 12K samples)" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } } ] }, "contributions": { "claimed": [ { "text": { "value": "INTELLECT-3: state-of-the-art performance in its weight class, achieving 90.8% / 88.0% on AIME 2024/2025 (outperforming DeepSeek's frontier models and matching GLM-4.6, which has over 3× the parameters) and 69.3% on LiveCodeBench v6 (+8% over GLM-4.5-Air post-train)", "provenance": "stated", "ref": "Abstract, §1, Table 2" }, "ref": { "value": "Abstract, §1, Table 2", "provenance": "stated", "ref": "Abstract, §1, Table 2" } }, { "text": { "value": "prime-rl: an open, production-scale asynchronous RL framework (FSDP2 trainer + vLLM inference, disaggregated and individually multi-node deployable) with continuous batching and in-flight weight updates, a multi-client orchestrator with round-robin distribution, online difficulty filtering, and efficient MoE support", "provenance": "stated", "ref": "§1, §2.1" }, "ref": { "value": "§1, §2.1", "provenance": "stated", "ref": "§1, §2.1" } }, { "text": { "value": "verifiers and the Environments Hub: a unified, versionable environment interface and open registry that standardize agentic RL training and evaluation, decoupling environments from the trainer", "provenance": "stated", "ref": "§1, §2.2" }, "ref": { "value": "§1, §2.2", "provenance": "stated", "ref": "§1, §2.2" } }, { "text": { "value": "Prime Sandboxes: high-throughput secure code execution for RL (Rust-based gateway bypassing the Kubernetes control plane, gVisor runtime, image streaming + warm pools, 256 sandboxes per node) supporting over 4,000 concurrent sandboxes during training", "provenance": "stated", "ref": "§2.3, §3.1.2" }, "ref": { "value": "§2.3, §3.1.2", "provenance": "stated", "ref": "§2.3, §3.1.2" } }, { "text": { "value": "Distributed Muon for FSDP-sharded gradients via an all-to-all-based shard reshuffle, avoiding InfiniBand congestion from overlapping gathers", "provenance": "stated", "ref": "§2.1.7" }, "ref": { "value": "§2.1.7", "provenance": "stated", "ref": "§2.1.7" } }, { "text": { "value": "RL recipe: masked token-level importance sampling (IcePop-style) with double-sided masking and a 1e-5 token-ratio floor, found critical for stable long-context agentic RL; scaling RL to 512 H200s with high training efficiency", "provenance": "stated", "ref": "§3.3, Abstract" }, "ref": { "value": "§3.3, Abstract", "provenance": "stated", "ref": "§3.3, Abstract" } }, { "text": { "value": "Open-sourcing the complete stack — model, RL framework, environments, and recipe — so the same code spans single-node experiments to production-scale training", "provenance": "stated", "ref": "Abstract, §5" }, "ref": { "value": "Abstract, §5", "provenance": "stated", "ref": "Abstract, §5" } } ], "assessment": { "value": "INTELLECT-3 contributes no new model architecture: the report states plainly that it is an SFT+RL post-train of Z.ai's GLM-4.5-Air base, and every block-level attribute (attention, norm, activation, tokenizer, expert routing) is inherited and undisclosed. Its genuine novelty is the open RL infrastructure — prime-rl's disaggregated async trainer/inference with in-flight weight updates and multi-client orchestration, the verifiers/Environments Hub ecosystem, and Prime Sandboxes' control-plane-bypassing execution path — plus the RL algorithm detail of double-sided masked token-level importance sampling (α=0.5, β=5, 1e-5 floor) with ablations against GSPO/CISPO. vs deepseek-r1 (the atlas's other open RLVR post-train): INTELLECT-3 is built on an MoE base instead of R1's dense V3-base, uses Muon rather than AdamW, and emphasizes multi-turn agentic environments (SWE, deep research) alongside math/science. vs openai-gpt-oss: both are open post-trained reasoning MoEs, but INTELLECT-3's base (GLM-4.5-Air, ~355B-class MoE family) and full infrastructure disclosure differ. The headline claims (SOTA in weight class, matching GLM-4.6) rest on the report's own harness (Appendix A) and are not independently verifiable here; the report itself flags that its AIME numbers are conservative (no LLM judge).", "provenance": "inferred", "note": "comparison against atlas entries (zai-glm-4-5-air, deepseek-r1, openai-gpt-oss, deepseek-v3-2) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "stated Mixture-of-Experts model, 106B total / 12B active (Abstract, §2.1)" }, "sequence_mixing": { "value": "full-attention", "provenance": "inferred", "note": "attention internals not described (inherited from GLM-4.5-Air); evidence for standard full attention: Flash Attention 3 usage and the attention-score-matrix memory analysis with context parallelism / ring attention discussion (§2.1.6); no sparse/linear/SSM mechanism is mentioned anywhere" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no non-text modality described)" }, "openness": { "value": "open-weights-open-data", "provenance": "derived", "note": "model open-sourced (stated) and the full post-training data/recipe is open: SFT datasets are permissive open-source collections (Table 1) and all RL environments and their datasets are released on the Environments Hub (stated §1, §5)" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "106B ≥ 100B (reference total params)" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "framed as a reasoning/agentic specialist: RLVR across math, code, science, logic, deep research and SWE; always-reasoning chat template; 'state of the art in its weight class' on reasoning benchmarks (Abstract, §1, §3.2)" } }, "prose": { "design_overview": { "value": "INTELLECT-3 is a 106B-parameter sparse MoE (12B active, 46 layers, hidden 4096, per-expert MoE dim 1408) that Prime Intellect built entirely by post-training Z.ai's GLM-4.5-Air base — the report discloses no block-level detail of its own, since every architectural attribute is inherited. The contribution is the training system around that fixed block. Two SFT stages establish behavior: a general chat-and-reasoning stage (~33M tokens/step at 65K context, Muon, warmup to 5e-5) and a smaller agentic stage at 98K context via context parallelism, using a Qwen3/GLM-style chat template that always prepends a <|think|> token and preserves reasoning across turns. Then a large-scale RLVR stage (batch 256 × 16 rollouts, 65,536 max context) optimizes the model across math, code, science, logic, deep-research (Serper search tools) and software-engineering environments with verifiable rewards — rule-based math-verify plus a CompassVerifier-7B LLM judge, sandbox test execution, and binary search rewards. The RL loop is the engineering centerpiece: prime-rl runs trainer (FSDP2) and inference (vLLM) disaggregated, with continuous batching, in-flight weight updates (max_off_policy_steps 8), a multi-client orchestrator for linear inference scaling, online difficulty filtering, and an IcePop-style double-sided masked importance-sampling objective (α=0.5, β=5) that the authors found critical for stable long-horizon training. Execution happens in Prime Sandboxes (gVisor containers, Rust gateway, image streaming, up to 256 sandboxes per node) at 4,000+ concurrent rollouts, on a 512-H200 cluster with distributed all-to-all Muon. The report frames the model as evidence for an open, reproducible RL stack as much as for the model itself.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Base model and full architecture: SFT and RL run on top of the GLM-4.5-Air base (all block-level design inherited)", "provenance": "stated", "ref": "Abstract, §1, §3 ('We use GLM-4.5-Air base as our base model')" }, "origin": { "value": "GLM-4.5-Air (Z.ai)", "provenance": "stated", "ref": "§1, §3" } }, { "mechanism": { "value": "Masked token-level importance sampling objective (J_IcePop) with double-sided masking and group-relative advantages", "provenance": "stated", "ref": "§3.3 (cites IcePop [55])" }, "origin": { "value": "IcePop (not in atlas)", "provenance": "stated", "ref": "§3.3" } }, { "mechanism": { "value": "Muon optimizer reuse in post-training (justified by the base model having been pretrained with Muon)", "provenance": "stated", "ref": "§2.1.7 (cites [25])" }, "origin": { "value": "Muon (not in atlas)", "provenance": "stated", "ref": "§2.1.7" } }, { "mechanism": { "value": "Chat template with <|system|>/<|user|>/<|assistant|> and <|im_start|>/<|im_end|> control tokens, XML-style tool calls, <|think|> reasoning token", "provenance": "stated", "ref": "§3.2 ('inspired by Qwen3- and GLM-family of models')" }, "origin": { "value": "Qwen3 / GLM chat templates", "provenance": "stated", "ref": "§3.2" } } ], "influence_out": [], "notable_omissions": { "value": [ "No expert count or experts-per-token disclosed (Figure 5's expert sweep is a kernel benchmark, not the model config)", "No attention configuration (heads, KV heads, MLA vs GQA), position encoding, normalization, activation, or tokenizer — the entire block is inherited from GLM-4.5-Air and never restated", "No vocab size, no embedding-tying statement", "No license named and no release date printed (undated PDF, non-arXiv)", "Deployed context length not stated (training max 65,536 RL / 98K agentic SFT)", "No pretraining (post-train only) and no compute (FLOPs/GPU-hours) disclosure", "Training precision not stated", "No quantization, KV-cache, or serving-optimization discussion", "No load-balancing method stated (MaxViolation is only a logged diagnostic)", "MMLU-Pro 83.5 for GLM-4.5 is marked 'reported by AA Index' rather than measured in-house (Table 2 footnote)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "prime-intellect-intellect-3.json" }, { "slug": { "value": "qwen-qwen3-5", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2604.15804", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/qwen-qwen3-5/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Qwen3.5", "provenance": "stated", "ref": "§2.3 (Qwen3.5 tokenizer); §2.5 (Hybrid MoE architecture introduced in Qwen3.5); §3 (S1: LLM initialized with parameters from Qwen3.5); §5.1.1 (Qwen3.5-Plus-Instruct)", "note": "the report's title names the Omni variant ('Qwen3.5-Omni Technical Report'); 'Qwen3.5' is how the report names the text family whose tokenizer, vision encoder, Hybrid MoE backbone, base checkpoint and Plus-Instruct model it references" }, "variants": [ { "value": "Qwen3.5-Omni-Plus", "provenance": "stated", "ref": "§1 ('The model series includes Plus and Flash variants, all of which are instruct models with 256k-token long-context input'); Abstract" }, { "value": "Qwen3.5-Omni-Flash", "provenance": "stated", "ref": "§1; Table 2" }, { "value": "Qwen3.5-Plus-Instruct", "provenance": "stated", "ref": "§5.1.1 (text-only counterpart used as baseline in Table 4)" } ], "org": { "value": "Qwen Team", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-04-21", "provenance": "stated", "ref": "arXiv header line: 'arXiv:2604.15804v2 [cs.CL] 21 Apr 2026'", "note": "date printed in the report header (v2 submission); the arXiv API (export.arxiv.org) was unreachable on 2026-08-11 so the v1 date could not be cross-checked" }, "report_url": { "value": "https://arxiv.org/abs/2604.15804", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "CC BY 4.0", "provenance": "stated", "ref": "arXiv HTML header: 'License: CC BY 4.0'", "note": "this is the paper license displayed on the arXiv page; the report never names a model-weights license" }, "open_weights": { "value": null, "provenance": "unknown", "note": "report contains no statement about releasing model weights (no release/availability/HuggingFace mention anywhere in the source)" } }, "scale": { "total_params": { "value": null, "provenance": "unknown", "note": "reference variant per schema §3 is the report's flagship Qwen3.5-Omni-Plus; no parameter count is disclosed for any variant — the Abstract states only that Qwen3.5-Omni 'scales to hundreds of billions of parameters'. The gallery-card figure '397B-A17B' never appears in the report." }, "active_params_per_token": { "value": null, "provenance": "unknown", "note": "no active-parameter figure disclosed ('A17B' never appears in the report)" }, "layers": { "value": null, "provenance": "unknown", "note": "no layer count disclosed for any model" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "no hidden dimension disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "no FFN dimension disclosed" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "no FFN dimensions disclosed, so no ratio can be derived" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "no attention head configuration disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "no KV-head/GQA/MLA information disclosed" }, "head_dim": { "value": null, "provenance": "unknown", "note": "no head dims disclosed" }, "vocab_size": { "value": 250000, "provenance": "stated", "ref": "§2.3", "note": "Qwen3.5 tokenizer used by the Thinker for text: 'a vocabulary size of 250k (up from 150k)'" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "input/output embedding tying not disclosed" }, "variants": [ { "name": { "value": "Qwen3.5-Omni-Plus", "provenance": "stated", "ref": "§1; Abstract" }, "total_params": { "value": null, "provenance": "unknown", "note": "only 'hundreds of billions of parameters' (Abstract); not a config-table figure" }, "layers": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "context_length": { "value": 262144, "provenance": "stated", "ref": "Abstract ('supports a 256k context length'); §1 ('256k-token long-context input')" } }, { "name": { "value": "Qwen3.5-Omni-Flash", "provenance": "stated", "ref": "§1; Table 2" }, "total_params": { "value": null, "provenance": "unknown", "note": "not disclosed; Table 2 notes 'substantial scale difference' vs Plus but never quantifies it" }, "layers": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "context_length": { "value": 262144, "provenance": "stated", "ref": "§1 (series-wide '256k-token long-context input')" } }, { "name": { "value": "Qwen3.5-Plus-Instruct (text-only counterpart)", "provenance": "stated", "ref": "§5.1.1 (Table 4 baseline column)" }, "total_params": { "value": null, "provenance": "unknown", "note": "not disclosed; §1 says only that text counterparts are 'same-size' as the Omni models" }, "layers": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "context_length": { "value": null, "provenance": "unknown", "note": "the 256k figure is stated for the Omni series, not for the text model" } } ] }, "block": { "block_type": { "value": "hybrid", "provenance": "derived", "note": "report labels the backbone 'Hybrid Mixture-of-Experts (MoE)' (§2.1) and 'the Hybrid MoE architecture introduced in Qwen3.5' (§2.5); per-layer dense/MoE split is not disclosed, so the ≥80% layer-majority rule cannot be applied — the report's own hybrid label governs" }, "moe": { "expert_count": { "value": null, "provenance": "unknown", "note": "no expert counts disclosed" }, "experts_per_token": { "value": null, "provenance": "unknown", "note": "no routing sparsity disclosed" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "routing": { "value": null, "provenance": "unknown", "note": "router function never described" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no load-balancing mechanism described" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "'Hybrid MoE' is named (§2.1, §2.5) but no router, expert organization or granularity is described" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§1 ('Hybrid-Attention Mixture-of-Experts (MoE) framework'); §2.5 (GDN module)", "note": "report names the framework 'Hybrid-Attention MoE' and states the Qwen3.5 architecture 'includes the Gated Delta Net (GDN) module'; the full-attention component is not otherwise described" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "per-layer arrangement of GDN vs full-attention layers not disclosed" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: no cross-layer mixing mechanism (Attention Residuals / Hyper-Connections style) described" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "inferred", "note": "TM-RoPE is applied for audio-video temporal awareness (§2.3, §3), implying a RoPE-based positional-encoding family; the base text position encoding is not described in detail" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE base frequency not disclosed" }, "partial_rope": { "value": false, "provenance": "inferred", "note": "absence rule: no partial-RoPE scheme described" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "S3 pre-training stage trains at maximum length 262,144 (§3), equal to the deployed 256k context (§1); no YaRN/NTK/PI-style extension described" }, "factor": { "value": null, "provenance": "n/a", "ref": "§3 (no extension method applies; training occurs at the deployed length)" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§1 ('256k-token long-context input'); §3 (S3 length 262,144)" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "RMSNorm/LayerNorm never mentioned in the report" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "absence rule: no QK-normalisation described on attention layers" } }, "activation": { "value": null, "provenance": "unknown", "note": "activation function never named (no SwiGLU/GELU/etc. mention)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: no designed sink mechanism described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described" }, "other": [] } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§3 (S3: 'we increased the maximum token length from 32,768 to 262,144')", "note": "stated for the Qwen3.5-Omni variant (the report's flagship, per schema §3); the text model's own training length is not disclosed" }, "deployed_length": { "value": 262144, "provenance": "stated", "ref": "Abstract ('supports a 256k context length'); §1 ('256k-token long-context input')", "note": "stated for the Qwen3.5-Omni series; the text model's context length is not disclosed" }, "extension_method": { "value": "progressive pre-training curriculum: sequence length raised from 32,768 (S2 general stage) to 262,144 (S3 long-context stage); training occurs at the deployed length, so no post-hoc extension method is described", "provenance": "stated", "ref": "§3" } }, "tokenizer": { "algorithm": { "value": "byte-level byte-pair encoding (BPE)", "provenance": "stated", "ref": "§2.3 ('adopts byte-level byte-pair encoding')" }, "notes": { "value": "Qwen3.5 tokenizer with vocabulary 250k up from 150k; report claims 10–60% encoding/decoding efficiency improvement across most languages", "provenance": "stated", "ref": "§2.3" } }, "training": { "tokens": { "value": 4, "provenance": "stated", "ref": "§3 (S2: 'approximately 4 trillion tokens')", "note": "≈4T tokens for the Omni model's S2 general stage — text 0.92T, audio 1.99T, image 0.95T, video 0.14T, video-audio 0.29T; the Qwen3.5 text backbone's own pre-training token budget is not disclosed" }, "data_composition": { "value": "omnimodal corpus: image-text, video-text, audio-text, video-audio, video-audio-text and pure-text corpora; more than 100 million hours of audio-visual data; AuT audio encoder trained on 40M hours of audio-text pair data generated by Qwen3-ASR; natural-language prompt diversity following Qwen3-Omni; text covers 201 languages/dialects (complete list deferred to the Qwen3.5 report), speech input 113 varieties, speech output 36 varieties", "provenance": "stated", "ref": "§1, §2.2, §3, Table 3" }, "curriculum": { "value": "three-stage pre-training: S1 encoder alignment (LLM locked and initialized from Qwen3.5, vision encoder from Qwen3.5, AuT audio encoder; adapters trained before encoders); S2 general stage, all parameters unfrozen, sequence length 32,768; S3 long-context stage, sequence length 262,144 with a higher proportion of long audio/video data", "provenance": "stated", "ref": "§3" }, "optimizer": { "value": null, "provenance": "unknown", "note": "no optimizer, learning rate, batch size or schedule disclosed for any stage" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "no learning-rate schedule disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "no batch sizes disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision (BF16/FP8) not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "no training parallelization strategy disclosed (deployment-side notes only: internal vLLM, torch.compile, CUDA Graph; §2.5, Table 2)" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1 (Stage 1: specialist teachers trained via independent Supervised Fine-Tuning (SFT) and RL; ChatML-format corpus)", "note": "described for the Omni Thinker; the teacher models are fine-tuned from the pre-trained Qwen-3.5 base checkpoint" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§4.1 Stage 3 (Interaction-Aligned RL, algorithm unnamed); §4.2 (Talker: DPO + GSPO with rule-based rewards)", "note": "the Thinker's RL stages are not tied to a named algorithm (RLHF/DPO/GRPO never named for the Thinker); DPO and GSPO are named only for the Talker speech pipeline" }, "reasoning_training": { "value": "specialist teacher models for agentic, coding and foundational reasoning tasks trained with SFT + RL and distilled into a single unified model; on-policy distillation (OPD) aligns audio-conditioned responses to higher-quality text-conditioned targets; interaction-aligned RL optimizes multi-turn interaction quality (language code-switching, persona consistency, instruction following)", "provenance": "stated", "ref": "§4.1" }, "distillation": { "value": "specialist distillation (domain-specialized teachers — agentic, coding, reasoning plus vision and audio — distilled into one unified model) and on-policy distillation (OPD) from text-conditioned to audio-conditioned responses; teachers fine-tuned from the pre-trained Qwen-3.5 base checkpoint", "provenance": "stated", "ref": "§4.1" } }, "modality": { "type": { "value": "text-only", "provenance": "stated", "ref": "§5.1.1 ('Qwen3.5-Plus-Instruct' as the text-only counterpart, Table 4)", "note": "the record covers the text-only Qwen3.5 model (gallery card); the report's own subject, Qwen3.5-Omni, is multimodal — AuT audio encoder (6.25 Hz tokens), SigLIP2 vision encoder, RVQ audio tokenizer, Code2Wav ConvNet (Table 1, §2)" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§5.1.1 (record is the text-only model)", "note": "no non-text attachment applies to the text-only model; the Omni variant's attachments are listed in modality.type note" } }, "efficiency": { "kv_cache": { "value": "hybrid-attention cache: the Gated Delta Net (GDN) module is credited with reducing KV-cache I/O overhead in long-context inference (§2.5); per-layer cache structure and sizes not disclosed", "provenance": "derived", "note": "derived from attention_variant (hybrid) plus the stated GDN KV-cache benefit" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "absence finding: no quantization scheme mentioned anywhere in the report" }, "speculative_dedup": { "value": "MTP (multi-token prediction) module — a Dense Transformer (Table 1) that predicts residual RVQ codebook tokens frame-by-frame for the Talker's streaming speech generation; no speculative decoding described for the text/Thinker path", "provenance": "stated", "ref": "§2.4, Table 1" }, "serving": { "value": "chunked pre-filling of audio/vision encoder outputs; streaming Thinker input and streaming Talker; internal vLLM with torch.compile and CUDA Graph acceleration for the MTP module and codec decoder; first-packet latency Plus 435 ms (audio) / 651 ms (video), Flash 235 ms / 426 ms; Thinker TPS 57–266 (Plus) and 177–942 (Flash) across 1–8 concurrency", "provenance": "stated", "ref": "§2.5, Table 2" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 86.8, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Redux", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 94.3, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SuperGPQA", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 67.4, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "C-Eval", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 92.3, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "IFEval", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 89.7, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "IFBench", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 51.1, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 62.0, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LongBench v2", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 60.2, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 85.9, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LiveCodeBench v6", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 67.1, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HMMT Nov 25", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 86.2, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "IMOAnswerBench", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 68.3, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "BFCL-V4", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 66.1, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "TAU2Bench", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 82.7, "provenance": "stated", "ref": "Table 4 (Qwen3.5-Plus-Instruct column)" }, "ref": { "value": "Table 4 (Qwen3.5-Plus-Instruct column)", "provenance": "stated", "ref": "Table 4" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Scaling the omnimodal Thinker–Talker architecture to hundreds of billions of parameters with 256k-token context, using Hybrid-Attention MoE (including the GDN module) for both Thinker and Talker to enable efficient long-sequence inference", "provenance": "stated", "ref": "Abstract; §1; §2.5" }, "ref": { "value": "Abstract; §1; §2.5", "provenance": "stated", "ref": "Abstract; §1; §2.5" } }, { "text": { "value": "ARIA (Adaptive Rate Interleave Alignment): a single-stream interleaving of text and speech tokens under an adaptive rate constraint, replacing dual-track generation to stabilize streaming speech synthesis and improve prosody with minimal latency impact", "provenance": "stated", "ref": "Abstract; §2.4" }, "ref": { "value": "Abstract; §2.4", "provenance": "stated", "ref": "Abstract; §2.4" } }, { "text": { "value": "Expanded multilingual speech: 113 languages/dialects for speech recognition and 36 for speech synthesis, with zero-shot voice customization from user-provided samples", "provenance": "stated", "ref": "§1; Table 3" }, "ref": { "value": "§1; Table 3", "provenance": "stated", "ref": "§1; Table 3" } }, { "text": { "value": "Controllable audio-visual captioning: script-level structured captions with timestamp annotation, character-relationship description, and automated scene segmentation", "provenance": "stated", "ref": "Abstract; §1" }, "ref": { "value": "Abstract; §1", "provenance": "stated", "ref": "Abstract; §1" } }, { "text": { "value": "Native omnimodal agentic behavior: autonomous WebSearch, complex FunctionCall, and Audio-Visual Vibe Coding — emergent direct code generation from audio-visual instructions", "provenance": "stated", "ref": "Abstract; §1" }, "ref": { "value": "Abstract; §1", "provenance": "stated", "ref": "Abstract; §1" } }, { "text": { "value": "Replacing TMRoPE temporal IDs with explicit text-string timestamps and contiguous cross-modality position numbering for robust long audio-video temporal perception", "provenance": "stated", "ref": "§2.3; §3" }, "ref": { "value": "§2.3; §3", "provenance": "stated", "ref": "§2.3; §3" } } ], "assessment": { "value": "The genuinely novel surface of this report sits entirely on the Omni side — ARIA's adaptive-rate single-stream text/speech interleaving, timestamp-string temporal encoding replacing TMRoPE, and multi-codebook MTP streaming — none of which concerns the text backbone this record represents. The text-side disclosure reduces to a name: 'Hybrid MoE' with a GDN module, i.e. a dense/MoE hybrid containing a gated delta-rule linear-attention component, consistent with the hybrid-attention trend already in the atlas (qwen-qwen3-next's hybrid backbone; moonshot-kimi-k3's KDA/MLA hybrid). Because the report never documents the text model's configuration, its novelty relative to Qwen3-Next-style or DeepSeek-style hybrids cannot be assessed from this source. The claim that the Omni model matches 'same-size' text counterparts on text benchmarks (§1, Table 4) is internal to the report and not independently verifiable.", "provenance": "inferred", "note": "authored by atlas analyst; comparison against atlas entries (qwen-qwen3-next, qwen-qwen3-moe, qwen-qwen3-dense, moonshot-kimi-k3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "hybrid", "provenance": "derived", "note": "report labels the backbone 'Hybrid MoE' / 'Hybrid Mixture-of-Experts (MoE)' (§2.1, §2.5); per-layer dense/MoE split undisclosed, so the ≥80% layer-majority rule cannot be applied — the report's own hybrid label governs" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "'Hybrid-Attention MoE' is stated (§1, Conclusion) and the architecture 'includes the Gated Delta Net (GDN) module' (§2.5) — full attention plus a delta-rule linear-attention component; per-layer composition undisclosed — the report's own hybrid label governs" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type: the record is the text-only Qwen3.5 model (gallery card); the report explicitly frames Qwen3.5-Plus-Instruct as the 'text-only counterpart' (§5.1.1), while the Omni variants it documents are multimodal" }, "openness": { "value": "undisclosed", "provenance": "derived", "note": "open_weights is unknown — the report never states whether weights are released; training-data composition is described (for the Omni variant) but no dataset release is mentioned; 'License: CC BY 4.0' on the arXiv page is the paper license, not a weights license" }, "scale_class": { "value": "frontier", "provenance": "inferred", "note": "Abstract: the Omni model 'scales to hundreds of billions of parameters'; §1: text counterparts are 'same-size' — hundreds of billions ≥ 100B; no exact count disclosed for any model" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "report frames the family as a general-purpose agentic assistant ('native omni agent model'); text evaluation spans knowledge, instruction following, long context, STEM, reasoning and general agent tasks (§5.1.1); the Omni variant adds real-time omnimodal interaction (Thinker–Talker, ARIA)" } }, "prose": { "design_overview": { "value": "This record is a coverage caveat in itself: the report behind it is the Qwen3.5-Omni Technical Report, and it never documents the Qwen3.5 text backbone that this record's gallery card names (397B-A17B). Neither '397B' nor 'A17B' appears anywhere in the text; the only scale statement is that the Omni model 'scales to hundreds of billions of parameters'. What the report does disclose about the Qwen3.5 family: a 250k-vocabulary byte-level BPE tokenizer; a backbone it names 'Hybrid MoE' — 'Hybrid-Attention Mixture-of-Experts' — introduced in Qwen3.5 and reused for both the Thinker and Talker of the Omni system; and the fact that this architecture 'includes the Gated Delta Net (GDN) module', a delta-rule-style linear-attention component credited with cutting KV-cache I/O in long-context inference. The text-only counterpart Qwen3.5-Plus-Instruct serves as the evaluation baseline (Table 4) and is said to be 'same-size' as the Omni models. Around that skeleton the report details the Omni variant: Thinker–Talker streaming with a SigLIP2 vision encoder, an AuT audio encoder trained on 40M hours of audio-text data, RVQ multi-codebook speech with an MTP module and Code2Wav renderer, ARIA's adaptive text-speech interleaving, explicit timestamp tokens replacing TMRoPE for temporal grounding, and a three-stage pre-training (encoder alignment, then a 32,768-token general stage on ~4T tokens, then a 262,144-token long-context stage) followed by specialist distillation, on-policy distillation, and interaction-aligned RL. Everything about the text model's own layers, width, expert routing, context length, and training remains undisclosed; per schema §3, the scale/context/training leaves above describe the report's flagship (Qwen3.5-Omni-Plus) where stated.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Thinker–Talker dual-component architecture (Thinker generates text; Talker generates streaming speech conditioned on Thinker representations)", "provenance": "stated", "ref": "§2.1 ('continues to adopt the Thinker-Talker architecture (Xu et al., 2025a)')" }, "origin": { "value": "Qwen2.5-Omni", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "RVQ-based speech representation (multi-codebook codec tokens)", "provenance": "stated", "ref": "§2.1 ('adopts the RVQ-based speech representation introduced in Qwen3-Omni')" }, "origin": { "value": "Qwen3-Omni", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Chunked pre-filling (encoders output temporal chunks to cut Time-To-First-Token)", "provenance": "stated", "ref": "§2.5 ('retain the chunked-prefilling mechanism as implemented in Qwen3-Omni and Qwen2.5-Omni')" }, "origin": { "value": "Qwen3-Omni / Qwen2.5-Omni", "provenance": "stated", "ref": "§2.5" } }, { "mechanism": { "value": "TM-RoPE temporal position encoding (applied, then superseded by explicit text-string timestamps with contiguous cross-modality position numbering)", "provenance": "stated", "ref": "§2.3, §3 ('Following Qwen3-Omni, we apply TM-RoPE')" }, "origin": { "value": "Qwen3-Omni", "provenance": "stated", "ref": "§2.3, §3" } }, { "mechanism": { "value": "ChatML conversation format for the training corpus", "provenance": "stated", "ref": "§4.1, §4.2 ('structured in the ChatML (OpenAI, 2022) format')" }, "origin": { "value": "OpenAI", "provenance": "stated", "ref": "§4.1" } }, { "mechanism": { "value": "LLM initialization and vision encoder adopted from the Qwen3.5 text family (S1 encoder-alignment stage initializes the Omni LLM with Qwen3.5 parameters)", "provenance": "stated", "ref": "§3 (S1: 'the LLM component of Qwen3.5-Omni is initialized with parameters from Qwen3.5, while the vision encoder is adopted from Qwen3.5')" }, "origin": { "value": "Qwen3.5 (text family)", "provenance": "stated", "ref": "§3" } } ], "influence_out": [], "notable_omissions": { "value": [ "CRITICAL COVERAGE GAP: the report documents the Qwen3.5-Omni multimodal variant only — the Qwen3.5 text backbone (the 397B-A17B model of the gallery card) is never described; '397B'/'A17B' never appear, and no layers, hidden dim, heads, expert counts, activation, normalization, context length, or training details are given for the text model", "No configuration table for any variant (no total/active parameters, layers, dims, heads, vocab aside from the 250k tokenizer, context)", "Only scale statement for any model: 'hundreds of billions of parameters' (Abstract, Omni); the Plus vs Flash sizes are contrasted ('substantial scale difference', Table 2) but never quantified", "Expert count, router, and load balancing of the MoE never described — 'Hybrid MoE' is a name, not a spec", "GDN (Gated Delta Net) is named as part of the Qwen3.5 architecture but its mechanism is never explained", "No optimizer, learning rate, batch size, precision, parallelism, hardware, or compute (FLOPs/GPU-hours) disclosed for any training stage", "No statement on weight release or a model-weights license (CC BY 4.0 on the arXiv page is the paper license only)", "Text model context length not disclosed (the 256k figure is stated for the Omni series); tokenizer disclosure limited to byte-level BPE + 250k vocab", "No attention head/KV design (GQA/MLA), head dim, or embedding tying disclosed", "Per-component context limits not reconciled: the Talker's long-context stage extends to 64k tokens (§4.2) while the Thinker reaches 262,144 (§3)", "Pretraining corpus is described for the Omni model only (≈4T tokens, S2); the text model's own data mixture is never disclosed" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "qwen-qwen3-5.json" }, { "slug": { "value": "qwen-qwen3-dense", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2505.09388", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/qwen-qwen3-dense/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Qwen3", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "Qwen3-0.6B", "provenance": "stated", "ref": "§2, Table 1" }, { "value": "Qwen3-1.7B", "provenance": "stated", "ref": "§2, Table 1" }, { "value": "Qwen3-4B", "provenance": "stated", "ref": "§2, Table 1" }, { "value": "Qwen3-8B", "provenance": "stated", "ref": "§2, Table 1" }, { "value": "Qwen3-14B", "provenance": "stated", "ref": "§2, Table 1" }, { "value": "Qwen3-32B", "provenance": "stated", "ref": "§2, Table 1" } ], "org": { "value": "Qwen Team", "provenance": "stated", "ref": "Title page byline (huggingface.co/Qwen, github.com/QwenLM/Qwen3)" }, "release_date": { "value": "2025-05-15", "provenance": "stated", "ref": "Title page date line (2025-05-15); arXiv stamp reads 14 May 2025" }, "report_url": { "value": "https://arxiv.org/abs/2505.09388", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "Abstract ('all Qwen3 models are publicly accessible under Apache 2.0')" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract, §1 ('open-weight large language models'; 'publicly accessible under Apache 2.0')" } }, "scale": { "total_params": { "value": 32, "provenance": "stated", "ref": "Table 4 (# Total Params 32B); model name Qwen3-32B" }, "active_params_per_token": { "value": 32, "provenance": "derived", "note": "dense model: active params = total params (32B)" }, "layers": { "value": 64, "provenance": "stated", "ref": "Table 1 (Qwen3-32B)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "report never discloses hidden (embedding) dimensions for any Qwen3 model; Table 1 lists only layers, heads, tie-embedding, context length" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "report never discloses FFN intermediate dimensions" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: hidden_dim not disclosed" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 1 (Heads Q/KV: 64 / 8 for Qwen3-32B)" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1 (Heads Q/KV: 64 / 8 for Qwen3-32B)" }, "head_dim": { "value": null, "provenance": "unknown", "note": "cannot derive: hidden_dim not disclosed" }, "vocab_size": { "value": 151669, "provenance": "stated", "ref": "§2 ('a vocabulary size of 151,669')" }, "embedding_tied": { "value": false, "provenance": "stated", "ref": "Table 1 (Tie Embedding: No for Qwen3-32B; Yes only for 0.6B/1.7B/4B)" }, "variants": [ { "name": { "value": "Qwen3-0.6B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 0.6, "provenance": "stated", "ref": "Table 8 (# Total Params 0.6B); model name" }, "layers": { "value": 28, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 16, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 32768, "provenance": "derived", "note": "Table 1 prints 32K; K = 1024 per the 32,768-token stage-3 training length in §3.2" } }, { "name": { "value": "Qwen3-1.7B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 1.7, "provenance": "stated", "ref": "Table 8 (# Total Params 1.7B); model name" }, "layers": { "value": 28, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 16, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 32768, "provenance": "derived", "note": "Table 1 prints 32K; K = 1024 per §3.2" } }, { "name": { "value": "Qwen3-4B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 4, "provenance": "stated", "ref": "Table 7 (# Total Params 4B); model name" }, "layers": { "value": 36, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 131072, "provenance": "derived", "note": "Table 1 prints 128K; K = 1024 per §3.2 (32,768 = 32K); 128K = 131,072" } }, { "name": { "value": "Qwen3-8B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 8, "provenance": "stated", "ref": "Table 6 (# Total Params 8B); model name" }, "layers": { "value": 36, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 131072, "provenance": "derived", "note": "Table 1 prints 128K; K = 1024 per §3.2; 128K = 131,072" } }, { "name": { "value": "Qwen3-14B", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 14, "provenance": "stated", "ref": "Table 5 (# Total Params 14B); model name" }, "layers": { "value": 40, "provenance": "stated", "ref": "Table 1" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 40, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 131072, "provenance": "derived", "note": "Table 1 prints 128K; K = 1024 per §3.2; 128K = 131,072" } } ] }, "block": { "block_type": { "value": "dense", "provenance": "derived", "note": "all layers of the dense models use fully active FFNs; no routing anywhere (report describes dense models as standard FFN stacks, §2)" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "§2 (dense models; no MoE)" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "§2 (dense models; no MoE)" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "§2 (dense models; no MoE)" }, "routing": { "value": null, "provenance": "n/a", "ref": "§2 (dense models; no MoE)" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "§2 (dense models; no MoE)" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "§2 (dense models; no MoE)" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2 (Grouped Query Attention, GQA)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "report describes GQA for the whole dense stack with no per-layer pattern; Qwen2.5-similar architecture (§2)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: architecture described as Qwen2.5-similar standard residual transformer stack (§2); no attention-over-layers mechanism mentioned" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2 (Rotary Positional Embeddings)" }, "rope_base": { "value": 1000000, "provenance": "stated", "ref": "§3.2 ('we increase the base frequency of RoPE from 10,000 to 1,000,000 using the ABF technique')" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "report silent on RoPE dimension coverage" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "note": "YaRN with scaling factor 4 plus Dual Chunk Attention (DCA); RoPE base raised to 1,000,000 via ABF; four-fold increase in sequence-length capacity at inference (§3.2, A.1.1 'we utilize YARN with a scaling factor=4')", "ref": "§3.2, §A.1.1" }, "factor": { "value": 4, "provenance": "stated", "ref": "§A.1.1 ('YARN with a scaling factor=4'); §3.2 ('a four-fold increase in sequence length capacity during inference')" }, "target_length": { "value": 131072, "provenance": "derived", "note": "32,768 × 4 (YARN/DCA extension of the 32,768-token stage-3 training length); Table 1 lists context length 128K for Qwen3-32B" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2 (RMSNorm with pre-normalization)" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§2 ('RMSNorm with pre-normalization')" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2 ('we remove QKV-bias used in Qwen2 and introduce QK-Norm (Dehghani et al., 2023) to the attention mechanism to ensure stable training')" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§2" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "report silent; architecture description is brief (component list), not detailed enough for the absence rule" }, "softcapping": { "value": null, "provenance": "unknown", "note": "report silent on logit softcapping" }, "other": { "value": null, "provenance": "unknown", "note": "no other stability mechanisms disclosed beyond QK-Norm (recorded in normalization.qk_norm)" } } }, "context": { "trained_length": { "value": 32768, "provenance": "stated", "ref": "§3.2 stage 3 ('All models are pre-trained on hundreds of billions of tokens with a sequence length of 32,768 tokens')" }, "deployed_length": { "value": 131072, "provenance": "derived", "note": "Table 1 lists Context Length 128K for Qwen3-32B; 128 × 1024 = 131,072 (K = 1024 per §3.2's 32,768-token length); consistent with the 4× YARN/DCA extension of 32,768" }, "extension_method": { "value": "YaRN + Dual Chunk Attention (DCA) with ABF base-frequency increase to 1,000,000; four-fold inference length extension", "provenance": "stated", "ref": "§3.2" } }, "tokenizer": { "algorithm": { "value": "byte-level BPE (BBPE)", "provenance": "stated", "ref": "§2 ('Qwen's tokenizer ... implements byte-level byte-pair encoding (BBPE)')" }, "notes": { "value": "Qwen tokenizer, vocabulary 151,669; chat template adds /think and /no_think flags and / tokens for thinking-mode control (Table 9)", "provenance": "stated", "ref": "§2, §4.3" } }, "training": { "tokens": { "value": 36, "provenance": "stated", "ref": "§1, §3.1 ('a total of 36 trillion tokens')" }, "data_composition": { "value": "119 languages and dialects; domains include coding, STEM, reasoning tasks, books, multilingual texts, and synthetic data; PDF-like documents transcribed by Qwen2.5-VL with Qwen2.5 refinement; trillions of synthetic tokens generated by Qwen2.5, Qwen2.5-Math, Qwen2.5-Coder (textbooks, QA, instructions, code); instance-level data-mixture optimization via a multilingual annotation system applied to over 30 trillion tokens (educational value, fields, domains, safety)", "provenance": "stated", "ref": "§3.1" }, "curriculum": { "value": "three-stage pre-training: S1 general stage >30T tokens at sequence length 4,096; S2 reasoning stage ~5T higher-quality tokens at 4,096 with accelerated learning-rate decay; S3 long-context stage hundreds of billions of tokens at 32,768 (75% 16,384–32,768-token texts, 25% 4,096–16,384)", "provenance": "stated", "ref": "§3.2" }, "optimizer": { "value": null, "provenance": "unknown", "note": "report never names the optimizer" }, "lr_schedule": { "value": "learning-rate decay accelerated during S2; optimal learning rate per model predicted via scaling laws (values not printed)", "provenance": "stated", "ref": "§3.2" }, "batch_schedule": { "value": "batch size strategy predicted via scaling laws for each dense/MoE model (values not printed)", "provenance": "stated", "ref": "§3.2" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "parallelization strategy not disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed; only relative GPU-hour figures appear for the Qwen3-8B distillation-vs-RL comparison (17,920 vs 1,800, Table 21)" }, "compute": { "value": null, "provenance": "unknown", "note": "no total FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1 (long-CoT cold-start SFT), §4.3 (continual SFT in Thinking Mode Fusion)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§4.2 ('employed GRPO (Shao et al., 2024) to update the model parameters'); Stage 4 General RL adds rule-based and model-based rewards" }, "reasoning_training": { "value": "long-CoT cold start (responses generated by QwQ-32B, two-phase query/response filtering); Reasoning RL on 3,995 query-verifier pairs (math/code/STEM); thinking budget mechanism halting thinking at a user-defined threshold; /think and /no_think mode switching with / tokens", "provenance": "stated", "ref": "§4.1–4.3" }, "distillation": { "value": "Strong-to-Weak Distillation for smaller models (0.6B/1.7B/4B/8B/14B dense and 30B-A3B MoE): off-policy response distillation from teacher outputs, then on-policy distillation aligning student logits with teacher (Qwen3-32B or Qwen3-235B-A22B) via KL divergence; ~1/10 the GPU hours of four-stage RL", "provenance": "stated", "ref": "§4.5, §4.7 (Table 21)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes text-only LLMs; Qwen2.5-VL appears only as a pre-training data-extraction tool (§3.1), not as a Qwen3 modality" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§2 (text-only architecture)" } }, "efficiency": { "kv_cache": { "value": "GQA: KV cache scales with 8 KV heads (Qwen3-32B: 64 Q heads / 8 KV heads)", "provenance": "derived", "note": "from attention_variant GQA; KV heads = 8 per Table 1" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence (YaRN deployment to 64K context appears only in BFCL evaluation settings, §4.6)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (5-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 83.61, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Pro (5-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 65.54, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SuperGPQA (5-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 39.78, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "BBH (3-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 87.38, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA (5-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 49.49, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GSM8K (4-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 93.4, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MATH (4-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 61.62, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "EvalPlus (0-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 72.05, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MultiPL-E (0-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 67.06, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MBPP (3-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 78.2, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "CRUX-O (1-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 72.5, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MGSM (8-shot, CoT)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 83.06, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMMLU (5-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 83.83, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "INCLUDE (5-shot)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 67.87, "provenance": "stated", "ref": "Table 4 (Qwen3-32B Base)" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Redux (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 90.9, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "GPQA-Diamond (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 68.4, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "AIME'24 (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 81.4, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "AIME'25 (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 72.9, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "LiveBench 2024-11-25 (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 74.9, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "Arena-Hard (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 93.8, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "BFCL v3 (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 70.3, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "LiveCodeBench v5 (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 65.7, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "CodeForces Rating (Thinking)", "provenance": "stated", "ref": "Table 13" }, "value": { "value": 1977, "provenance": "stated", "ref": "Table 13 (Qwen3-32B Thinking; CodeForces Rating / Percentile 1977 / 97.7%)" }, "ref": { "value": "Table 13", "provenance": "stated", "ref": "Table 13" } }, { "name": { "value": "RULER 128K (Non-thinking)", "provenance": "stated", "ref": "Table 23" }, "value": { "value": 85.6, "provenance": "stated", "ref": "Table 23 (Qwen3-32B Non-thinking, 128K column)" }, "ref": { "value": "Table 23", "provenance": "stated", "ref": "Table 23" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Integration of thinking mode and non-thinking mode into a single model with dynamic mode switching via chat templates, eliminating the need to alternate between chat and reasoning models", "provenance": "stated", "ref": "Abstract, §1, §4.3" }, "ref": { "value": "Abstract, §1, §4.3", "provenance": "stated", "ref": "Abstract, §1, §4.3" } }, { "text": { "value": "Thinking budget mechanism giving users fine-grained control over reasoning effort (token budget) at inference, with performance scaling smoothly with the budget", "provenance": "stated", "ref": "Abstract, §1, §4.7" }, "ref": { "value": "Abstract, §1, §4.7", "provenance": "stated", "ref": "Abstract, §1, §4.7" } }, { "text": { "value": "Strong-to-Weak Distillation (off-policy plus on-policy logit distillation) for lightweight models, achieving better performance than RL at about 1/10 of the GPU hours", "provenance": "stated", "ref": "§4.5, §4.7" }, "ref": { "value": "§4.5, §4.7", "provenance": "stated", "ref": "§4.5, §4.7" } }, { "text": { "value": "Multilingual expansion from 29 to 119 languages and dialects, with instance-level data-mixture optimization via a large-scale annotation system", "provenance": "stated", "ref": "Abstract, §3.1" }, "ref": { "value": "Abstract, §3.1", "provenance": "stated", "ref": "Abstract, §3.1" } }, { "text": { "value": "Architecture improvements over Qwen2.5: QK-Norm added to the attention mechanism and QKV-bias removed for stable training", "provenance": "stated", "ref": "§2" }, "ref": { "value": "§2", "provenance": "stated", "ref": "§2" } }, { "text": { "value": "Scaling laws developed for optimal hyperparameters (learning-rate scheduler, batch size) across the three pre-training stages", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } } ], "assessment": { "value": "The dense block is a conservative Qwen2.5-lineage stack (GQA + SwiGLU + RoPE + RMSNorm pre-norm) with one structural addition, QK-Norm, and one removal, QKV-bias; closest atlas relative is meta-llama-3.1 (same four-mechanism recipe, also RoPE + pre-RMSNorm), from which Qwen3 dense differs mainly by QK-Norm, the 1,000,000 RoPE base with YaRN+DCA 4× extension instead of direct 128K training, and a tie-embedding policy split by size (small models tie, 8B+ do not). The genuinely novel contribution is the post-training pipeline (long-CoT cold start → reasoning RL with GRPO → thinking-mode fusion → general RL) and strong-to-weak distillation for small models; these are training recipes, not new mixing mechanisms. The report discloses no hidden dims or FFN dims (unlike Llama 3.1 / Phi-3 / DeepSeek-V3 reports), so per-variant width comparisons in the atlas are impossible from this source.", "provenance": "inferred", "note": "comparison against atlas entries (meta-llama-3.1, deepseek-v3, microsoft-phi-3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "derived", "note": "all layers of the dense models are fully active FFNs; no routing (§2)" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "all layers use unrestricted GQA attention over the whole context; no sliding window, linear attention, or SSM layers (§2)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Apache 2.0); data composition described but not released" }, "scale_class": { "value": "large", "provenance": "derived", "note": "32B reference variant: 10B ≤ 32B < 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a general-purpose open-weight foundation family with state-of-the-art results across code, math, agents, and multilingual tasks (Abstract, §1); thinking mode is an integrated feature rather than the sole framing. Secondary intents: reasoning-specialised emphasis (thinking mode, §4) noted in prose." } }, "prose": { "design_overview": { "value": "Qwen3 dense is the standard-bearer of the Qwen2.5 architectural line: each layer is a pre-RMSNorm transformer block with GQA attention, SwiGLU feed-forward, and RoPE, stacked 64 deep in the reference Qwen3-32B (36 layers in 4B/8B, 40 in 14B, 28 in 0.6B/1.7B). The two structural changes over Qwen2.5 are the removal of QKV-bias and the addition of QK-Norm to the attention mechanism, both aimed at training stability. Context is grown in three pre-training stages (4,096 → 4,096 → 32,768 tokens, 36T tokens total across 119 languages), then extended four-fold at inference via YaRN with scaling factor 4 plus Dual Chunk Attention, on top of an ABF base-frequency increase from 10,000 to 1,000,000; the 32B deploys at 128K. Post-training is where Qwen3 makes its mark: a long-CoT cold start with QwQ-32B-generated reasoning traces, GRPO-based reasoning RL on 3,995 verifiable query-verifier pairs, a thinking-mode fusion stage that fuses /think and /no_think behaviors into one model with a budget-controllable thinking process, and a general RL stage with rule-based and model-based rewards. The five smaller dense siblings (0.6B–14B) skip most of this: they are trained via strong-to-weak distillation, first off-policy on teacher outputs and then on-policy against teacher logits, at roughly a tenth of the RL compute.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Grouped Query Attention (GQA) — shared KV heads across query groups", "provenance": "stated", "ref": "§2 (cites Ainslie et al., 2023)" }, "origin": { "value": "GQA (Ainslie et al., 2023)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "SwiGLU gated activation", "provenance": "stated", "ref": "§2 (cites Dauphin et al., 2017)" }, "origin": { "value": "Dauphin et al., 2017", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Rotary Positional Embeddings (RoPE)", "provenance": "stated", "ref": "§2 (cites Su et al., 2024)" }, "origin": { "value": "Su et al., 2024", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "RMSNorm with pre-normalization", "provenance": "stated", "ref": "§2 (cites Jiang et al., 2023)" }, "origin": { "value": "Jiang et al., 2023", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "QK-Norm for attention-logit normalization", "provenance": "stated", "ref": "§2 (cites Dehghani et al., 2023)" }, "origin": { "value": "Dehghani et al., 2023", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Adjusted Base Frequency (ABF) RoPE base increase to 1,000,000", "provenance": "stated", "ref": "§3.2 (cites Xiong et al., 2023)" }, "origin": { "value": "Xiong et al., 2023", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "YaRN length extrapolation and Dual Chunk Attention (DCA)", "provenance": "stated", "ref": "§3.2, §A.1.1 (cites Peng et al., 2023; An et al., 2024)" }, "origin": { "value": "Peng et al., 2023 (YaRN); An et al., 2024 (DCA)", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "GRPO for reasoning RL", "provenance": "stated", "ref": "§4.2 (cites Shao et al., 2024)" }, "origin": { "value": "DeepSeekMath (Shao et al., 2024)", "provenance": "stated", "ref": "§4.2" } }, { "mechanism": { "value": "Overall dense block design 'similar to Qwen2.5'", "provenance": "stated", "ref": "§2" }, "origin": { "value": "Qwen2.5 (not in atlas)", "provenance": "stated", "ref": "§2" } } ], "influence_out": [ { "model_slug": { "value": "jetbrains-mellum2-thinking-12b-a2-5b", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Qwen3-MoE recipe — 'closely follows the Qwen3-MoE recipe' (MoE-in-every-layer, GQA, QK-Norm, RMSNorm, SiLU, RoPE); scaled down from Qwen3-30B-A3B", "provenance": "inferred", "note": "mechanism as stated in jetbrains-mellum2-thinking-12b-a2-5b report's lineage_in" } }, { "model_slug": { "value": "prime-intellect-intellect-3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Chat template with <|system|>/<|user|>/<|assistant|> and <|im_start|>/<|im_end|> control tokens, XML-style tool calls, <|think|> reasoning token", "provenance": "inferred", "note": "mechanism as stated in prime-intellect-intellect-3 report's lineage_in" } } ], "notable_omissions": { "value": [ "Hidden/embedding dimensions and FFN intermediate dimensions are never disclosed for any variant (Table 1 lists only layers, heads, tie-embedding, context length) — per-variant width comparisons are impossible from this report", "Training optimizer, precision, parallelism strategy, hardware, and total compute (FLOPs/GPU-hours) are not disclosed", "Learning rate and batch size values are not printed (only that they were scaling-law-predicted)", "No head dimension disclosed", "MoE record fields dependent on per-expert dims are unknown (this report's MoE section does not give them either)", "Source extraction: arxiv.org/html/2505.09388 is a truncated conversion ending at §4.3 (Thinking Mode Fusion) with Tables 10–37 missing; the stored source was re-extracted from the full 35-page PDF (116K chars), and PDF table columns interleave row-wise in the plain text (verified against context before use)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "qwen-qwen3-dense.json" }, { "slug": { "value": "qwen-qwen3-moe", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2505.09388", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/qwen-qwen3-moe/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Qwen3", "provenance": "stated", "ref": "Title" }, "variants": [ { "value": "Qwen3-30B-A3B", "provenance": "stated", "ref": "§2, Table 2" }, { "value": "Qwen3-235B-A22B", "provenance": "stated", "ref": "§2, Table 2" } ], "org": { "value": "Qwen Team", "provenance": "stated", "ref": "Title page byline (huggingface.co/Qwen, github.com/QwenLM/Qwen3)" }, "release_date": { "value": "2025-05-15", "provenance": "stated", "ref": "Title page date line (2025-05-15); arXiv stamp reads 14 May 2025" }, "report_url": { "value": "https://arxiv.org/abs/2505.09388", "provenance": "stated", "ref": "pipeline" }, "license": { "value": "Apache 2.0", "provenance": "stated", "ref": "Abstract ('all Qwen3 models are publicly accessible under Apache 2.0')" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract, §1 ('open-weight large language models'; 'publicly accessible under Apache 2.0')" } }, "scale": { "total_params": { "value": 235, "provenance": "stated", "ref": "§1, §2 (flagship 'has a total of 235B parameters'); Table 3 (# Total Params 235B)" }, "active_params_per_token": { "value": 22, "provenance": "stated", "ref": "§1 ('22 billion activated ones per token'); Table 3 (# Activated Params 22B)" }, "layers": { "value": 94, "provenance": "stated", "ref": "Table 2 (Qwen3-235B-A22B)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "report never discloses hidden (embedding) dimensions for any Qwen3 model; Table 2 lists only layers, heads, expert counts, context length" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "report never discloses FFN intermediate dimensions (per-expert or otherwise)" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: hidden_dim not disclosed" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 2 (Heads Q/KV: 64 / 4 for Qwen3-235B-A22B)" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "Table 2 (Heads Q/KV: 64 / 4 for Qwen3-235B-A22B)" }, "head_dim": { "value": null, "provenance": "unknown", "note": "cannot derive: hidden_dim not disclosed" }, "vocab_size": { "value": 151669, "provenance": "stated", "ref": "§2 ('a vocabulary size of 151,669')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Table 2 (MoE configs) has no Tie Embedding column and the text never states tying for the MoE models" }, "variants": [ { "name": { "value": "Qwen3-30B-A3B", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 30, "provenance": "stated", "ref": "Table 5 (# Total Params 30B); model name" }, "layers": { "value": 48, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 32, "provenance": "stated", "ref": "Table 2" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "Table 2" }, "context_length": { "value": 131072, "provenance": "derived", "note": "Table 2 prints 128K; K = 1024 per §3.2's 32,768-token stage-3 length (32K); 128K = 131,072" } }, { "name": { "value": "Qwen3-235B-A22B", "provenance": "stated", "ref": "Table 2" }, "total_params": { "value": 235, "provenance": "stated", "ref": "Table 3 (# Total Params 235B); model name" }, "layers": { "value": 94, "provenance": "stated", "ref": "Table 2" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 2" }, "attention_heads_kv": { "value": 4, "provenance": "stated", "ref": "Table 2" }, "context_length": { "value": 131072, "provenance": "derived", "note": "Table 2 prints 128K; K = 1024 per §3.2's 32,768-token stage-3 length (32K); 128K = 131,072" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "every transformer layer of both MoE variants uses a routed fine-grained MoE FFN; no dense layers anywhere in the MoE models (§2, Table 2)" }, "moe": { "expert_count": { "value": 128, "provenance": "stated", "ref": "§2 ('128 total experts'); Table 2 (# Experts Total / Activated: 128 / 8)" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§2 ('8 activated experts per token'); Table 2 (# Experts Total / Activated: 128 / 8)" }, "shared_experts": { "value": 0, "provenance": "stated", "ref": "§2 ('Unlike Qwen2.5-MoE, the Qwen3-MoE design excludes shared experts')" }, "routing": { "value": null, "provenance": "unknown", "note": "report never describes the router function (no gating/softmax/sigmoid detail); only the load-balancing loss is named" }, "load_balancing": { "value": "global-batch load balancing loss (Qiu et al., 2025) to encourage expert specialization", "provenance": "stated", "ref": "§2" }, "expert_granularity": { "value": "fine-grained expert segmentation following Qwen2.5-MoE (Dai et al., 2024)", "provenance": "stated", "ref": "§2 ('We follow Qwen2.5-MoE and implement fine-grained expert segmentation (Dai et al., 2024)')" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2 (MoE models 'share the same fundamental architecture as the Qwen3 dense models', which use Grouped Query Attention; Table 2 heads Q/KV 64/4 and 32/4)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "inferred", "note": "report describes GQA for the whole MoE stack with no per-layer pattern (§2)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: standard Qwen2.5-similar residual transformer stack (§2); no attention-over-layers mechanism mentioned" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2 (Rotary Positional Embeddings)" }, "rope_base": { "value": 1000000, "provenance": "stated", "ref": "§3.2 ('we increase the base frequency of RoPE from 10,000 to 1,000,000 using the ABF technique')" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "report silent on RoPE dimension coverage" }, "extension": { "method": { "value": "YaRN", "provenance": "stated", "note": "YaRN (scaling factor 4) plus Dual Chunk Attention (DCA); RoPE base raised to 1,000,000 via ABF; four-fold increase in sequence-length capacity at inference (§3.2, A.1.1)", "ref": "§3.2, §A.1.1" }, "factor": { "value": 4, "provenance": "stated", "ref": "§A.1.1 ('YARN with a scaling factor=4'); §3.2 ('a four-fold increase in sequence length capacity during inference')" }, "target_length": { "value": 131072, "provenance": "derived", "note": "32,768 × 4 (YARN/DCA extension of the 32,768-token stage-3 training length); Table 2 lists context length 128K for both MoE models" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2 (RMSNorm with pre-normalization)" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§2 ('RMSNorm with pre-normalization')" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2 (QK-Norm (Dehghani et al., 2023) introduced to the attention mechanism; 'The Qwen3 MoE models share the same fundamental architecture as the Qwen3 dense models')" } }, "activation": { "value": "SwiGLU", "provenance": "stated", "ref": "§2" }, "stability": { "attention_sinks": { "value": null, "provenance": "unknown", "note": "report silent; architecture description is a brief component list, not detailed enough for the absence rule" }, "softcapping": { "value": null, "provenance": "unknown", "note": "report silent on logit softcapping" }, "other": { "value": null, "provenance": "unknown", "note": "no other stability mechanisms disclosed beyond QK-Norm (recorded in normalization.qk_norm)" } } }, "context": { "trained_length": { "value": 32768, "provenance": "stated", "ref": "§3.2 stage 3 ('All models are pre-trained on hundreds of billions of tokens with a sequence length of 32,768 tokens')" }, "deployed_length": { "value": 131072, "provenance": "derived", "note": "Table 2 lists Context Length 128K for both MoE models; 128 × 1024 = 131,072 (K = 1024 per §3.2's 32,768-token length); consistent with the 4× YARN/DCA extension of 32,768" }, "extension_method": { "value": "YaRN + Dual Chunk Attention (DCA) with ABF base-frequency increase to 1,000,000; four-fold inference length extension", "provenance": "stated", "ref": "§3.2" } }, "tokenizer": { "algorithm": { "value": "byte-level BPE (BBPE)", "provenance": "stated", "ref": "§2 ('Qwen's tokenizer ... implements byte-level byte-pair encoding (BBPE)')" }, "notes": { "value": "Qwen tokenizer, vocabulary 151,669; chat template adds /think and /no_think flags and / tokens for thinking-mode control (Table 9)", "provenance": "stated", "ref": "§2, §4.3" } }, "training": { "tokens": { "value": 36, "provenance": "stated", "ref": "§1, §3.1 ('a total of 36 trillion tokens' for the whole Qwen3 family)" }, "data_composition": { "value": "119 languages and dialects; domains include coding, STEM, reasoning tasks, books, multilingual texts, and synthetic data; PDF-like documents transcribed by Qwen2.5-VL with Qwen2.5 refinement; trillions of synthetic tokens generated by Qwen2.5, Qwen2.5-Math, Qwen2.5-Coder (textbooks, QA, instructions, code); instance-level data-mixture optimization via a multilingual annotation system applied to over 30 trillion tokens (educational value, fields, domains, safety)", "provenance": "stated", "ref": "§3.1" }, "curriculum": { "value": "three-stage pre-training: S1 general stage >30T tokens at sequence length 4,096; S2 reasoning stage ~5T higher-quality tokens at 4,096 with accelerated learning-rate decay; S3 long-context stage hundreds of billions of tokens at 32,768 (75% 16,384–32,768-token texts, 25% 4,096–16,384)", "provenance": "stated", "ref": "§3.2" }, "optimizer": { "value": null, "provenance": "unknown", "note": "report never names the optimizer" }, "lr_schedule": { "value": "learning-rate decay accelerated during S2; optimal learning rate per model predicted via scaling laws (values not printed)", "provenance": "stated", "ref": "§3.2" }, "batch_schedule": { "value": "batch size strategy predicted via scaling laws for each dense/MoE model (values not printed)", "provenance": "stated", "ref": "§3.2" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "parallelization strategy not disclosed (no TP/PP/EP/CP details)" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed; only relative GPU-hour figures appear for the Qwen3-8B distillation-vs-RL comparison (17,920 vs 1,800, Table 21)" }, "compute": { "value": null, "provenance": "unknown", "note": "no total FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1 (long-CoT cold-start SFT), §4.3 (continual SFT in Thinking Mode Fusion)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§4.2 ('employed GRPO (Shao et al., 2024) to update the model parameters'); Stage 4 General RL adds rule-based and model-based rewards (§4.4)" }, "reasoning_training": { "value": "long-CoT cold start (responses generated by QwQ-32B, two-phase query/response filtering); Reasoning RL on 3,995 query-verifier pairs (math/code/STEM); thinking budget mechanism halting thinking at a user-defined threshold; /think and /no_think mode switching with / tokens", "provenance": "stated", "ref": "§4.1–4.3" }, "distillation": { "value": "Qwen3-30B-A3B: strong-to-weak distillation (off-policy response distillation from teacher outputs, then on-policy distillation aligning student logits with teacher Qwen3-32B or Qwen3-235B-A22B via KL divergence; ~1/10 the GPU hours of four-stage RL). Flagship Qwen3-235B-A22B is not distilled — it runs the full four-stage post-training pipeline.", "provenance": "stated", "ref": "§4.5, §4.7 (Table 21)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes text-only LLMs; Qwen2.5-VL appears only as a pre-training data-extraction tool (§3.1), not as a Qwen3 modality" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§2 (text-only architecture)" } }, "efficiency": { "kv_cache": { "value": "GQA: KV cache scales with 4 KV heads (Qwen3-235B-A22B: 64 Q / 4 KV heads; Qwen3-30B-A3B: 32 Q / 4 KV heads)", "provenance": "derived", "note": "from attention_variant GQA; KV heads = 4 per Table 2" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence (YaRN deployment to 64K context appears only in BFCL evaluation settings, §4.6)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU (5-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 87.81, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU-Pro (5-shot, CoT)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 68.18, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BBH (3-shot, CoT)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 88.87, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA (5-shot, CoT)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 47.47, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MATH (4-shot, CoT)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 71.84, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "EvalPlus (0-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 77.6, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MBPP (3-shot)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 81.4, "provenance": "stated", "ref": "Table 3 (Qwen3-235B-A22B Base)" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MMLU (5-shot)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 81.38, "provenance": "stated", "ref": "Table 5 (Qwen3-30B-A3B Base)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MMLU-Pro (5-shot, CoT)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 61.49, "provenance": "stated", "ref": "Table 5 (Qwen3-30B-A3B Base)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MATH (4-shot, CoT)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 59.04, "provenance": "stated", "ref": "Table 5 (Qwen3-30B-A3B Base)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "EvalPlus (0-shot)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 71.45, "provenance": "stated", "ref": "Table 5 (Qwen3-30B-A3B Base)" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "AIME'24 (Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 85.7, "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking); abstract" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "AIME'25 (Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 81.5, "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking); abstract" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "GPQA-Diamond (Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 71.1, "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking)" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "Arena-Hard (Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 95.6, "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking)" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "LiveCodeBench v5 (Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 70.7, "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking); abstract" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "BFCL v3 (Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 70.8, "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking); abstract" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "CodeForces (Rating / Percentile, Thinking)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": "2056 / 98.2%", "provenance": "stated", "ref": "Table 11 (Qwen3-235B-A22B Thinking)" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "MMLU-Redux (Non-thinking)", "provenance": "stated", "ref": "Table 12" }, "value": { "value": 89.2, "provenance": "stated", "ref": "Table 12 (Qwen3-235B-A22B Non-thinking)" }, "ref": { "value": "Table 12", "provenance": "stated", "ref": "Table 12" } }, { "name": { "value": "Arena-Hard (Non-thinking)", "provenance": "stated", "ref": "Table 12" }, "value": { "value": 96.1, "provenance": "stated", "ref": "Table 12 (Qwen3-235B-A22B Non-thinking)" }, "ref": { "value": "Table 12", "provenance": "stated", "ref": "Table 12" } }, { "name": { "value": "MATH-500 (Non-thinking)", "provenance": "stated", "ref": "Table 12" }, "value": { "value": 91.2, "provenance": "stated", "ref": "Table 12 (Qwen3-235B-A22B Non-thinking)" }, "ref": { "value": "Table 12", "provenance": "stated", "ref": "Table 12" } }, { "name": { "value": "LiveCodeBench v5 (Non-thinking)", "provenance": "stated", "ref": "Table 12" }, "value": { "value": 35.3, "provenance": "stated", "ref": "Table 12 (Qwen3-235B-A22B Non-thinking)" }, "ref": { "value": "Table 12", "provenance": "stated", "ref": "Table 12" } }, { "name": { "value": "AIME'24 (Thinking)", "provenance": "stated", "ref": "Table 15" }, "value": { "value": 80.4, "provenance": "stated", "ref": "Table 15 (Qwen3-30B-A3B Thinking)" }, "ref": { "value": "Table 15", "provenance": "stated", "ref": "Table 15" } }, { "name": { "value": "MATH-500 (Thinking)", "provenance": "stated", "ref": "Table 15" }, "value": { "value": 98.0, "provenance": "stated", "ref": "Table 15 (Qwen3-30B-A3B Thinking)" }, "ref": { "value": "Table 15", "provenance": "stated", "ref": "Table 15" } }, { "name": { "value": "ZebraLogic (Thinking)", "provenance": "stated", "ref": "Table 15" }, "value": { "value": 89.5, "provenance": "stated", "ref": "Table 15 (Qwen3-30B-A3B Thinking)" }, "ref": { "value": "Table 15", "provenance": "stated", "ref": "Table 15" } }, { "name": { "value": "RULER (Avg., Non-thinking)", "provenance": "stated", "ref": "Table 23" }, "value": { "value": 95.0, "provenance": "stated", "ref": "Table 23 (Qwen3-235B-A22B Non-thinking)" }, "ref": { "value": "Table 23", "provenance": "stated", "ref": "Table 23" } }, { "name": { "value": "RULER (Avg., Non-thinking)", "provenance": "stated", "ref": "Table 23" }, "value": { "value": 91.6, "provenance": "stated", "ref": "Table 23 (Qwen3-30B-A3B Non-thinking)" }, "ref": { "value": "Table 23", "provenance": "stated", "ref": "Table 23" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Flagship MoE Qwen3-235B-A22B (235B total / 22B activated) achieves state-of-the-art overall performance among open-source models in both thinking and non-thinking modes and is highly competitive with closed-source leaders (OpenAI-o1, Gemini2.5-Pro, GPT-4o)", "provenance": "stated", "ref": "§1, §4.6" }, "ref": { "value": "§1, §4.6", "provenance": "stated", "ref": "§1, §4.6" } }, { "text": { "value": "MoE design with 128 total experts and 8 activated per token via fine-grained expert segmentation, excluding shared experts entirely (unlike Qwen2.5-MoE), balanced by a global-batch load balancing loss", "provenance": "stated", "ref": "§2" }, "ref": { "value": "§2", "provenance": "stated", "ref": "§2" } }, { "text": { "value": "Unified thinking / non-thinking modes in a single model with /think and /no_think chat-template flags, plus a thinking budget mechanism that adaptively controls inference-time reasoning compute", "provenance": "stated", "ref": "§1, §4.3" }, "ref": { "value": "§1, §4.3", "provenance": "stated", "ref": "§1, §4.3" } }, { "text": { "value": "Strong-to-weak distillation builds lightweight models (including the Qwen3-30B-A3B MoE) with better performance and ~1/10 the GPU hours of the four-stage RL pipeline", "provenance": "stated", "ref": "§4.5, §4.7" }, "ref": { "value": "§4.5, §4.7", "provenance": "stated", "ref": "§4.5, §4.7" } }, { "text": { "value": "Pre-trained on 36 trillion tokens covering 119 languages and dialects (up from 29 in Qwen2.5), with instance-level data-mixture optimization via a multilingual annotation system", "provenance": "stated", "ref": "§1, §3.1" }, "ref": { "value": "§1, §3.1", "provenance": "stated", "ref": "§1, §3.1" } }, { "text": { "value": "Qwen3 MoE base models reach performance similar to Qwen3 dense base models with only 1/5 of the activated parameters on the same pre-training data, and outperform Qwen2.5 MoE base models with less than 1/2 the activated parameters", "provenance": "stated", "ref": "§3.3" }, "ref": { "value": "§3.3", "provenance": "stated", "ref": "§3.3" } } ], "assessment": { "value": "Architecturally the MoE branch is conservative — a Qwen2.5-lineage GQA + SwiGLU + RoPE + pre-RMSNorm transformer (plus QK-Norm) with fine-grained routing. Against DeepSeek-V3, the closest atlas peer (671B total / 37B active, 256 routed experts with 1 shared expert and aux-loss-free bias balancing), Qwen3-MoE is the mirror-image design: fewer and fatter experts (128 total / 8 active), no shared expert at all, and an explicit global-batch load-balancing auxiliary loss rather than an aux-loss-free scheme. Both choices (fine-grained segmentation, global-batch loss) are borrowed and cited (Dai et al. 2024; Qiu et al. 2025), not new mechanisms. The genuinely distinctive contributions sit in post-training — thinking/non-thinking mode unification, the thinking budget, and strong-to-weak distillation — and in scale economics: 22B active parameters matching DeepSeek-V3's active count at 35% of its total parameters. The report's efficiency claims (1/5 activated params vs dense, 1/10 GPU hours for distillation) are internal comparisons, not independently verifiable.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, qwen-qwen3-dense, mistral-mixtral-8x7b) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "every layer of both MoE variants uses a routed fine-grained MoE FFN; no dense layers (§2, Table 2)" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "GQA softmax attention on all layers (94/94 for the reference variant); no linear-attention or SSM layers" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Apache 2.0); data composition described but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "235B ≥ 100B (reference variant Qwen3-235B-A22B)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a general-purpose open family (0.6B–235B) with unified thinking/non-thinking modes for broad tasks (coding, math, agents, multilingual); flagship competes with closed frontier models. Secondary intent: reasoning-specialised via thinking mode, noted in prose." } }, "prose": { "design_overview": { "value": "Qwen3's MoE branch pairs a deliberately conventional transformer backbone with an aggressive sparsity story. Qwen3-235B-A22B and Qwen3-30B-A3B share the dense branch's Qwen2.5-style stack — GQA attention with QK-Norm, SwiGLU, RoPE with the base raised to 1,000,000 via ABF, pre-RMSNorm — but replace every FFN with a fine-grained MoE: 128 total experts with 8 activated per token and, unlike Qwen2.5-MoE or DeepSeek-V3, no shared expert at all. The report discloses no hidden or per-expert dimensions, no router function, and no embedding tying, so the block's economics are only partially visible; what is stated is that load is balanced by a global-batch load-balancing auxiliary loss (Qiu et al., 2025). Context reaches 128K via YaRN + Dual Chunk Attention over the 32,768-token stage-3 training length. The 235B flagship activates 22B parameters per token — the same active count as DeepSeek-V3 at roughly a third of its total parameters — and the family's headline claims sit in post-training: one model that switches between thinking and non-thinking modes via /think and /no_think chat-template flags, a thinking-budget mechanism that halts reasoning at a user-defined threshold, and strong-to-weak distillation that builds the lightweight members (including the 30B-A3B MoE) at roughly 1/10 of the four-stage RL GPU hours. The flagship itself runs the full pipeline: long-CoT cold start, GRPO reasoning RL on 3,995 query-verifier pairs, thinking-mode fusion SFT, then general RL with rule-based and model-based rewards.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Fine-grained expert segmentation for MoE FFNs", "provenance": "stated", "ref": "§2 ('We follow Qwen2.5-MoE (Yang et al., 2024b) and implement fine-grained expert segmentation (Dai et al., 2024)')" }, "origin": { "value": "Qwen2.5-MoE", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "QK-Norm on the attention mechanism (QKV-bias removed)", "provenance": "stated", "ref": "§2 (cites Dehghani et al., 2023)" }, "origin": { "value": "Dehghani et al., 2023 (QK-Norm)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "RoPE base-frequency increase from 10,000 to 1,000,000 via ABF", "provenance": "stated", "ref": "§3.2 (following Qwen2.5; cites Xiong et al., 2023)" }, "origin": { "value": "Qwen2.5", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "YaRN context extension with Dual Chunk Attention (DCA)", "provenance": "stated", "ref": "§3.2 (following Qwen2.5; cites Peng et al., 2023; An et al., 2024)" }, "origin": { "value": "Qwen2.5", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "Global-batch load balancing loss for expert specialization", "provenance": "stated", "ref": "§2 (cites Qiu et al., 2025)" }, "origin": { "value": "Qiu et al., 2025 (not in atlas)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "GRPO for Reasoning RL", "provenance": "stated", "ref": "§4.2 (cites Shao et al., 2024)" }, "origin": { "value": "Shao et al., 2024 (GRPO)", "provenance": "stated", "ref": "§4.2" } } ], "influence_out": [], "notable_omissions": { "value": [ "Hidden/embedding dimensions and FFN (per-expert) dimensions never disclosed; Table 2 lists only layers, heads, expert counts, and context length", "Router function not described (only the global-batch load balancing loss is named)", "Input/output embedding tying not stated for the MoE models (Table 2 has no tie-embedding column)", "Optimizer not named", "Training precision, parallelization strategy, hardware, and total compute (FLOPs/GPU-hours) not disclosed", "No per-variant pre-training token allocation for the MoE models (36T total for the whole family)", "No quantisation formats, serving optimisations, or speculative-decoding strategy disclosed", "Shared-expert removal is asserted without ablation evidence", "Head dimension not derivable (hidden dim undisclosed)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "qwen-qwen3-moe.json" }, { "slug": { "value": "qwen-qwen3-next", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2603.00729", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/qwen-qwen3-next/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Qwen3-Coder-Next", "provenance": "stated", "ref": "Title ('Qwen3-Coder-Next Technical Report'); gallery card names it 'Qwen3 Next 80B-A3B'" }, "variants": [ { "value": "80B-A3B", "provenance": "stated", "ref": "Abstract ('80-billion-parameter model that activates only 3 billion parameters during inference'); Table 3 (Size 80A3)", "note": "Only size disclosed by this report; the 30B-A3B 'Flash' sibling is NOT covered here" } ], "org": { "value": "Qwen Team", "provenance": "stated", "ref": "Title page byline (line: 'Qwen Team')" }, "release_date": { "value": "2026-02-28", "provenance": "stated", "ref": "Title page date line ('arXiv:2603.00729v1 [cs.CL] 28 Feb 2026')" }, "report_url": { "value": "https://arxiv.org/abs/2603.00729", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "Only 'License: arXiv.org perpetual non-exclusive license' (the paper license) appears; no model license is named" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('We release both base and instruction-tuned open-weight versions'); §1 ('an open-weight language model')" } }, "scale": { "total_params": { "value": 80, "provenance": "stated", "ref": "Abstract, §1 ('80 billion total parameters'); Table 3 (Size 80A3)" }, "active_params_per_token": { "value": 3, "provenance": "stated", "ref": "Abstract, §1 ('activates only 3 billion per forward pass'); Table 3 (Size 80A3)" }, "layers": { "value": null, "provenance": "unknown", "note": "report has no configuration table and never discloses layer count (architecture inherited from Qwen3-Next base)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed anywhere in the report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed (no expert or FFN dimensions given)" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "cannot derive: hidden_dim not disclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed (only 'hybrid attention' is stated, §1)" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed; exact attention mechanisms unnamed" }, "head_dim": { "value": null, "provenance": "unknown", "note": "cannot derive: hidden_dim and head counts not disclosed" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "tokenizer and vocabulary never described" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "variants": [] }, "block": { "block_type": { "value": "hybrid", "provenance": "derived", "note": "report labels the architecture 'hybrid mixture-of-experts' (§6) and 'hybrid attention and Mixture-of-Experts' (§1); no per-layer dense/MoE pattern or layer counts are disclosed, so the ≥80% majority rule cannot be applied — the report's own hybrid label governs" }, "moe": { "expert_count": { "value": null, "provenance": "unknown", "note": "model is MoE (3B active of 80B total) but no expert counts are disclosed" }, "experts_per_token": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "shared_experts": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "routing": { "value": null, "provenance": "unknown", "note": "no router function described" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "not discussed" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "not discussed" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§1 ('based on Qwen3-Next with hybrid attention and Mixture-of-Experts'); §6 ('hybrid mixture-of-experts architecture')", "note": "exact per-layer mechanisms (full vs linear attention, GQA/MLA) are never named" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "report does not describe the per-layer attention pattern" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: report describes no depth-mixing mechanism (no learned attention over prior layer outputs, no connection matrix); block internals are inherited from Qwen3-Next and undisclosed" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "report never mentions RoPE or any positional-encoding scheme" }, "rope_base": { "value": null, "provenance": "unknown", "note": "RoPE never mentioned" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "RoPE never mentioned, so partial-RoPE status cannot be judged" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "no context-extension technique (YaRN/NTK/PI) disclosed; mid-training context growth 32,768 → 262,144 tokens is stated (§3.1.1, §3.2) but not tied to a PE method" }, "factor": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "target_length": { "value": null, "provenance": "unknown", "note": "not disclosed (262,144 is the stated mid-training context length, not an extension target)" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "no normalization layer named (no RMSNorm/LayerNorm discussion anywhere)" }, "placement": { "value": null, "provenance": "unknown", "note": "not described" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: report never describes QK-normalisation on any attention layer; attention internals are undisclosed" } }, "activation": { "value": null, "provenance": "unknown", "note": "no activation function named" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: report describes no designed sink mechanism; block internals are undisclosed (inherited from Qwen3-Next)" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no logit softcapping described anywhere in the report" }, "other": [] } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§3.1.1, §3.2 ('we expand the training context length from 32,768 tokens to 262,144 tokens'; 'we extend the context length beyond typical pretraining settings to 262,144 tokens')" }, "deployed_length": { "value": null, "provenance": "unknown", "note": "report does not state the served/extension context length" }, "extension_method": { "value": null, "provenance": "unknown", "note": "mid-training context growth 32,768 → 262,144 tokens is disclosed (§3.1.1, §3.2) but no extension technique is named and the deployed length is unknown" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "report never describes the tokenizer (FIM tokens and ChatML format are referenced, §3.1.4)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "mid-training runs on 'trillions of tokens' (§3.2) but no exact figure is given; base pretraining (Qwen3-Next) is outside this report's scope" }, "data_composition": { "value": "mid-training corpus primarily natural data with a smaller synthetic portion: (1) GitHub source code with language support expanded from 92 to 370 languages, file- and repository-level data (~600B repository-level tokens; repository concatenation via special tokens), (2) text–code grounding data from Common Crawl and math/programming/education domains, rewritten into normalized Markdown by Qwen3-Coder-480B-A35B-Instruct, (3) PR-based structured SWE tasks mined from GitHub, (4) synthetic single-turn QA and multi-turn agentic trajectories generated by multiple agent frameworks (SWE-agent, Mini-SWE-agent, OpenHands, Claude-Code, Qwen-Code, Terminus) with Qwen3-Coder-480B-A35B-Instruct as teacher, (5) a small instruction-following mix, (6) FIM data from Stack-V2 (chat-FIM and search-and-replace FIM); pretraining corpus updated through Sep 30, 2025", "provenance": "stated", "ref": "§3.1, §3.1.1, §3.1.2, §3.1.4" }, "curriculum": { "value": "staged pipeline: continued pretraining (mid-training) on code/agent data with context 32,768 → 262,144 tokens → SFT → specialization of four domain experts → expert distillation into one unified model → single-turn and multi-turn execution-verifiable RL; best-fit packing (BFP) for sample packing; masking of highly repetitive segments", "provenance": "stated", "ref": "§1, §3.2, §4" }, "optimizer": { "value": null, "provenance": "unknown", "note": "never mentioned" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "never mentioned" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "never mentioned" }, "precision": { "value": null, "provenance": "unknown", "note": "never mentioned" }, "parallelism": { "value": null, "provenance": "unknown", "note": "never mentioned" }, "hardware": { "value": null, "provenance": "unknown", "note": "MegaFlow on Alibaba Cloud Kubernetes (§2.2) is agentic-task execution infrastructure, not training hardware" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed; §6 only notes 'lower total training compute' relative to frontier proprietary models" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1 ('we first perform supervised fine-tuning (SFT)'); data from in-house corpora, verified agentic trajectories, and documentation-grounded QA, filtered by a Mini-SWE-agent-based user-simulator verifier and pairwise judging" }, "preference_optimization": { "value": "none disclosed", "provenance": "stated", "ref": "§4.1", "note": "pairwise judging (§4.1) ranks SFT candidate responses — data curation, not policy preference optimization; RL uses execution-based rewards; RLHF/DPO/GRPO/PPO are never named" }, "reasoning_training": { "value": "execution-verifiable RL in two regimes: single-turn RL on unit-test-verified coding tasks (competitive programming, library-usage, multilingual, secure coding) with majority-vote unit tests; multi-turn agentic RL on SWE tasks with trajectory-level completion rewards plus unfinished-trajectory and turn-level tool-format penalties and a reinforced reward-hacking blocker; long-horizon behavior emerged (average agent turns 50 → 130)", "provenance": "stated", "ref": "§4.2.3, §4.2.4" }, "distillation": { "value": "expert distillation: knowledge from domain-specialized experts (Web Development, User Experience, Single-turn RL/QA, Software Engineering) distilled into the unified SFT model", "provenance": "stated", "ref": "§4.2.5", "note": "Qwen3-Coder-480B-A35B-Instruct also serves as teacher for synthetic data generation (§3.1.2) — teacher-generated data, which the schema treats as data synthesis, noted here for context" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes no multimodal input for the model; VLMs are used only as judges for WebDev data curation (§4.2.1); §6 lists 'integrating visual capability into future agent models' as future work" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§6 (text-only; visual capability planned only for future models)" } }, "efficiency": { "kv_cache": { "value": null, "provenance": "unknown", "note": "cannot derive: attention variant is stated only as 'hybrid' (§1) and no cache design is disclosed" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report-wide absence", "note": "MegaFlow (§2.2) is rollout/execution infrastructure for agentic tasks, not a serving optimization" } }, "evaluation": { "benchmarks": [ { "name": { "value": "SWE-Bench Verified (SWE-Agent / MiniSWE-Agent / OpenHands)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "70.6 / 71.1 / 71.3", "provenance": "stated", "ref": "Table 3, §5.1" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Bench Multilingual (SWE-Agent / MiniSWE-Agent / OpenHands)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "62.8 / 56.2 / 64.3", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SWE-Bench Pro (SWE-Agent / MiniSWE-Agent)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": "42.7 / 38.7", "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Terminal-Bench 2.0 (Terminus2-xml / Terminus2-json / ClaudeCode / QwenCode)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": "34.2 / 36.2 / 30.9 / 25.8", "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "EvalPlus", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 86.56, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MultiPL-E", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 88.23, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "CRUXEval", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 95.88, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "LiveCodeBench (v6)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 58.93, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "OJBench", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 23.01, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "Codeforces", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 2100, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "FullStackBench-en", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 60.58, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "FullStackBench-zh", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 57.38, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Spider", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 83.66, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "BIRD-SQL", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 63.56, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Aider-Polyglot", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 66.2, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 87.73, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } }, { "name": { "value": "MMLU-Redux", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 91.18, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 80.52, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } }, { "name": { "value": "GPQA", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 74.49, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } }, { "name": { "value": "SuperGPQA", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 57.45, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } }, { "name": { "value": "HMMT25 Feb", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 70.21, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "HMMT25 Nov", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 75.57, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "AIME24", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 89.01, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "AIME25", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 83.07, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "AthenaBench-Mini (CTK / RCM / VSP / TAA / RMS / ATE)", "provenance": "stated", "ref": "Table 14" }, "value": { "value": "85.00 / 44.00 / 58.50 / 5.50 / 24.50 / 8.00", "provenance": "stated", "ref": "Table 14 (Ours row, greedy decoding)" }, "ref": { "value": "Table 14", "provenance": "stated", "ref": "Table 14" } }, { "name": { "value": "PrimeVul-Paired (Accuracy)", "provenance": "stated", "ref": "Table 15" }, "value": { "value": 48.33, "provenance": "stated", "ref": "Table 15 (Ours row, greedy decoding)" }, "ref": { "value": "Table 15", "provenance": "stated", "ref": "Table 15" } }, { "name": { "value": "SecCodeBench (Gen w/o Hint)", "provenance": "stated", "ref": "Table 16" }, "value": { "value": 61.2, "provenance": "stated", "ref": "Table 16" }, "ref": { "value": "Table 16", "provenance": "stated", "ref": "Table 16" } }, { "name": { "value": "SecCodeBench (Gen w/ Hint)", "provenance": "stated", "ref": "Table 16" }, "value": { "value": 69.5, "provenance": "stated", "ref": "Table 16" }, "ref": { "value": "Table 16", "provenance": "stated", "ref": "Table 16" } }, { "name": { "value": "SecCodeBench (Fix w/o Hint)", "provenance": "stated", "ref": "Table 16" }, "value": { "value": 76.4, "provenance": "stated", "ref": "Table 16" }, "ref": { "value": "Table 16", "provenance": "stated", "ref": "Table 16" } }, { "name": { "value": "SecCodeBench (Fix w/ Hint)", "provenance": "stated", "ref": "Table 16" }, "value": { "value": 83.7, "provenance": "stated", "ref": "Table 16" }, "ref": { "value": "Table 16", "provenance": "stated", "ref": "Table 16" } }, { "name": { "value": "CWEval (func@1)", "provenance": "stated", "ref": "Table 16" }, "value": { "value": 80.17, "provenance": "stated", "ref": "Table 16" }, "ref": { "value": "Table 16", "provenance": "stated", "ref": "Table 16" } }, { "name": { "value": "CWEval (func-sec@1)", "provenance": "stated", "ref": "Table 16" }, "value": { "value": 56.32, "provenance": "stated", "ref": "Table 16" }, "ref": { "value": "Table 16", "provenance": "stated", "ref": "Table 16" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Scaling agentic training rather than model size: a compact 80B-A3B model, trained on large-scale synthesized verifiable coding tasks with environment feedback, matches models with an order of magnitude more active compute on agentic benchmarks", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } }, { "text": { "value": "Large-scale task-synthesis pipeline: GitHub PR mining with constructed runnable Docker environments plus extension of open-source executable datasets (SWE-Smith, SWE-Flow, SWE-Rebench, Multi-SWE-RL), yielding ~800K verifiable SWE task instances across 9+ programming languages, with quality-assurance-agent filtering", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "MegaFlow, a fully cloud-native (Alibaba Cloud Kubernetes) orchestration system enabling production-scale parallel execution, evaluation, and data generation for agentic coding workloads", "provenance": "stated", "ref": "§2.2" }, "ref": { "value": "§2.2", "provenance": "stated", "ref": "§2.2" } }, { "text": { "value": "Repository-level mid-training recipe: context expanded 32,768 → 262,144 tokens, GitHub language coverage expanded 92 → 370 languages, ~600B repository-level tokens, best-fit packing, and masking of highly repetitive segments", "provenance": "stated", "ref": "§3.1.1, §3.2" }, "ref": { "value": "§3.1.1, §3.2", "provenance": "stated", "ref": "§3.1.1, §3.2" } }, { "text": { "value": "Tool-chat-template diversity training and the new XML-style qwen3_coder tool-calling format for string-heavy arguments, improving format-invariant tool use across IDE/CLI scaffolds", "provenance": "stated", "ref": "§4.2.2" }, "ref": { "value": "§4.2.2", "provenance": "stated", "ref": "§4.2.2" } }, { "text": { "value": "Expert specialization followed by distillation: Web Development, User Experience, Single-turn QA, and Software Engineering experts consolidated into one unified deployment model without expert routing", "provenance": "stated", "ref": "§4.2, §4.2.5" }, "ref": { "value": "§4.2, §4.2.5", "provenance": "stated", "ref": "§4.2, §4.2.5" } }, { "text": { "value": "Execution-verifiable RL beyond competitive programming (single-turn unit-test-verified coding) plus multi-turn agentic RL with a reinforced reward-hacking blocker for GitHub-based environments, eliminating agent exploits of future-commit leakage", "provenance": "stated", "ref": "§4.2.3, §4.2.4" }, "ref": { "value": "§4.2.3, §4.2.4", "provenance": "stated", "ref": "§4.2.3, §4.2.4" } }, { "text": { "value": "First comparative evaluation of a coding model against frontier models on cybersecurity benchmarks (AthenaBench, PrimeVul-Paired, SecCodeBench, CWEval)", "provenance": "stated", "ref": "§A.4" }, "ref": { "value": "§A.4", "provenance": "stated", "ref": "§A.4" } } ], "assessment": { "value": "Every claimed contribution is in the training recipe and data infrastructure — none is architectural: the hybrid-attention MoE backbone is inherited from the Qwen3-Next base and the report discloses no config details at all. vs atlas peers, the mid-training → SFT → multi-expert → distillation → execution-RL pipeline extends the Qwen2.5-Coder/Qwen3-Coder lineage (repository-level pretraining with special concatenation tokens is explicitly carried over from Qwen2.5-Coder), and tool-template diversity plus the reward-hacking blocker are practical engineering contributions of the same family as the agentic-RL recipes seen in Kimi K3, though K3's report is far more architecture-heavy. The genuinely distinctive claims are (1) the demonstration that a 3B-active model can match models with ~10× active compute via scaled agentic training, and (2) the first security-benchmark comparison for a coding agent. Architecturally this is the thinnest record in the atlas: layers, experts, attention internals, and normalization are simply not discussed.", "provenance": "inferred", "note": "authored by atlas analyst; comparison against atlas entries (qwen-qwen3-moe, qwen-qwen3-dense, moonshot-kimi-k3, deepseek-v3-2) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "hybrid", "provenance": "derived", "note": "report explicitly labels the architecture 'hybrid mixture-of-experts' (§6) and 'hybrid attention and Mixture-of-Experts' (§1); no per-layer dense/MoE pattern or layer counts are disclosed, so the ≥80% majority rule cannot be applied — the report's own hybrid label governs" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "'hybrid attention' is stated (§1) but the per-layer composition (full vs linear attention) is undisclosed; majority rule not applicable — the report's own label governs" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no multimodal input described; VLMs used only for data curation §4.2.1; visual capability listed as future work §6)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (Abstract); data composition described in detail (§3.1) but no dataset release disclosed" }, "scale_class": { "value": "large", "provenance": "derived", "note": "80B reference total params ∈ [10B, 100B)" }, "design_intent": { "value": "domain-specific", "provenance": "inferred", "note": "framed throughout as 'specialized for coding agents' and 'designed specifically for coding agents and local development' (§1, Abstract); general knowledge is preserved but secondary (Table 8); secondary intent: agentic/long-horizon tool use" } }, "prose": { "design_overview": { "value": "Qwen3-Coder-Next is an 80-billion-parameter mixture-of-experts model that activates 3 billion parameters per token, built on the Qwen3-Next base with a hybrid-attention backbone — and that is the report's entire architectural disclosure. There is no configuration table: layers, hidden width, head counts, expert counts, vocabulary, normalization, and activation are all inherited from the base model and left unspecified. What the report documents instead is a complete agentic training recipe. Mid-training expands the context from 32,768 to 262,144 tokens so the model can carry multi-turn tool-use trajectories, mixing natural GitHub data (370 languages, roughly 600B repository-level tokens) with synthetic PR-derived tasks (~800K verifiable instances across 9+ languages), text–code grounding data rewritten by the larger Qwen3-Coder-480B-A35B-Instruct teacher, and FIM data in chat and search-and-replace forms. After continued pretraining, SFT on verified agentic trajectories is followed by specialization into four domain experts — web development, user experience/tool-format adherence, single-turn QA, software engineering — and distillation back into a single unified model. Execution-verifiable RL then runs in two regimes: single-turn, unit-test-verified coding, and multi-turn agentic SWE interaction with trajectory-level rewards, token-level tool-format penalties, and a reinforced reward-hacking blocker. The result is a small-footprint coding agent that matches models with an order of magnitude more active compute on the SWE-Bench suites, arguing that scaling agentic training, not model size, drives coding-agent capability. Architecturally this record is the thinnest in the atlas; as a training-recipe disclosure it is among the most detailed.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Hybrid attention + MoE backbone (80B total / 3B active per token)", "provenance": "stated", "ref": "§1 ('based on Qwen3-Next with hybrid attention and Mixture-of-Experts')" }, "origin": { "value": "Qwen3-Next (pretrained base)", "provenance": "stated", "ref": "§1, §3" } }, { "mechanism": { "value": "Repository-level pretraining with special concatenation tokens", "provenance": "stated", "ref": "§3.1.1 ('Consistent with Qwen2.5-Coder, we use special tokens to concatenate repository data')" }, "origin": { "value": "Qwen2.5-Coder", "provenance": "stated", "ref": "§3.1.1" } } ], "influence_out": [], "notable_omissions": { "value": [ "No architecture/config table: layers, hidden dim, head counts, expert counts, FFN dims, vocab size, activation, and normalization are all undisclosed (inherited from Qwen3-Next)", "Exact attention composition not disclosed beyond 'hybrid attention' — no full/linear layer pattern, no GQA/MLA details, no attention heads", "No model license named (only the arXiv paper license line appears)", "No pretraining details: the Qwen3-Next base pretraining is out of scope; mid-training volume given only as 'trillions of tokens'", "No optimizer, learning rate, batch size, precision, parallelism, training hardware, or compute (FLOPs/GPU-hours) disclosed", "RL algorithm not named (no GRPO/PPO/RLOO); only 'execution-based rewards' are described", "Deployed context length and any positional-encoding extension method not disclosed", "Tokenizer not described (FIM tokens and ChatML format referenced only)", "No KV-cache, quantization, speculative-decoding, or serving disclosures (MegaFlow covers rollout infrastructure, not serving)", "The 30B-A3B 'Flash' sibling is not covered by this report" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "qwen-qwen3-next.json" }, { "slug": { "value": "upstage-solar-open-2", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.4.0", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://huggingface.co/upstage/Solar-Open2-250B/raw/main/Solar_Open_2_Tech_Report.pdf", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/upstage-solar-open-2/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "Solar Open 2", "provenance": "stated", "ref": "Title ('Solar Open 2 Technical Report')" }, "variants": [], "org": { "value": "Upstage (Upstage Solar Team)", "provenance": "stated", "ref": "Title page byline ('Upstage Solar Team'); website upstage.ai" }, "release_date": { "value": "2026-07-22", "provenance": "stated", "ref": "Title page ('Date: Jul 22, 2026'); arXiv:2607.20062v1 [cs.CL] 22 Jul 2026" }, "report_url": { "value": "https://huggingface.co/upstage/Solar-Open2-250B/raw/main/Solar_Open_2_Tech_Report.pdf", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named in the report" }, "open_weights": { "value": true, "provenance": "stated", "ref": "§1 ('an openly available, nationally grounded frontier model'); §4 (MOPD consolidates twelve teachers 'into the single released model'); Acknowledgments ('the development and release of Solar Open 2')" } }, "scale": { "total_params": { "value": 250, "provenance": "stated", "ref": "Abstract, §2, Table 1 ('250B total parameters')" }, "active_params_per_token": { "value": 15, "provenance": "stated", "ref": "Abstract, §2, Table 1 ('15B active parameters per token')" }, "layers": { "value": 48, "provenance": "stated", "ref": "§2, Table 1 ('Num Layers 48')" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "§2, Table 1 ('Hidden Size 4,096')" }, "ffn_inner_dim": { "value": 1280, "provenance": "stated", "ref": "Table 1 ('MoE Intermediate Size 1,280' — per-expert FFN inner dimension)" }, "ffn_ratio": { "value": 0.3125, "provenance": "derived", "note": "1280 / 4096 = 0.3125 (per-expert MoE intermediate size over hidden size)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "§2, Table 1 ('64 query and 8 key-value heads')" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1 ('Attention Heads (Q / KV) 64 / 8'), note a: 'KV heads apply only to the softmax layers; the linear layers use 64 heads and keep no KV cache'" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "§2, Table 1 ('Head Dimension 128')" }, "vocab_size": { "value": 196608, "provenance": "stated", "ref": "§2, Table 1 ('a 196,608-token vocabulary')" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "the token embedding and output layer are transferred together (same shape) but no tying is stated" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "every one of the 48 layers pairs attention with an MoE FFN block ('a single shared expert with no dense layers', §2; Figure 3 shows MoE FFN on all 48 layers)" }, "moe": { "expert_count": { "value": 320, "provenance": "stated", "ref": "§2, Table 1 ('Num Experts (Routed) 320')" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "Table 1 ('Experts per Token (Top-k) 8')" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "§2, Table 1 ('Num Shared Experts 1'; 'a single shared expert with no dense layers')" }, "routing": { "value": null, "provenance": "unknown", "note": "only the top-8 count is disclosed; the router function (softmax/sigmoid/etc.) is never described" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no load-balancing mechanism described" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "no expert grouping or latent-space design described; experts operate at full width with MoE intermediate size 1,280 (Table 1)" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.2, Table 1 ('Hybrid: [Softmax ×1, Linear ×3] ×12'; GQA softmax layers + KDA linear layers)" }, "attention_layer_pattern": { "value": "per block of four: one softmax (GQA) layer first, then three linear (KDA) layers — S-L-L-L; repeated 12 times; 12 softmax (25%) + 36 linear (75%) layers; softmax-first ordering explicitly contrasts with the L-L-L-S ordering of Kimi Linear and Qwen3.5", "provenance": "stated", "ref": "§2.2 ('interleaves one softmax-attention layer with three linear-attention layers in every block of four... so that 12 of the 48 layers (25%) are softmax and 36 (75%) are linear'; 'the softmax layer comes first (S-L-L-L)')" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: standard pre-norm residual stack; no attention-over-layers or learned connection matrix described" }, "position_encoding": { "method": { "value": "NoPE", "provenance": "stated", "ref": "§2.2 ('no positional encoding anywhere in the model'), Table 1 ('Positional Embedding: NoPE')" }, "rope_base": { "value": null, "provenance": "n/a", "ref": "§2.2 (no RoPE anywhere in the model; Solar Open 1's RoPE θ = 10⁶ is listed only as the discarded predecessor setting)" }, "partial_rope": { "value": null, "provenance": "n/a", "ref": "§2.2 (no RoPE)" }, "extension": { "method": { "value": "none", "provenance": "inferred", "note": "NoPE design needs no positional-encoding modification ('In principle, Solar Open 2 can therefore support an unbounded context length', §2.2); the 1M window is reached by staged training — Stage 4 Length Expansion trains at a 1M context length (§3.3)" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.2 (no rescaling factor applies; no RoPE)" }, "target_length": { "value": 1048576, "provenance": "stated", "ref": "Table 1 ('Context Length 1,048,576 (1M)')" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "only 'pre-norm residuals' and 'normalization layers' are mentioned; no RMSNorm/LayerNorm type named" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§2 ('pre-norm residuals')" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "QK-norm is not described; the L2-normalized keys inherent to KDA linear attention are part of the linear-attention parameterization, not evidence of QK-norm on global-attention layers" } }, "activation": { "value": "SiLU", "provenance": "stated", "ref": "Table 1 ('Activation: SiLU')" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: no designed sink tokens; the sigmoid output gate is described as suppressing the emergent 'attention sink' collapse (§2.2), which belongs in prose, not here" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no attention-logit softcapping described" }, "other": [ { "value": "Elementwise sigmoid output gate on the softmax layers' scaled dot-product attention output (query-dependent gating): introduces non-linearity into the low-rank attention mapping, makes outputs query-dependently sparse, suppresses attention-sink collapse, and tolerates larger learning rates", "provenance": "stated", "ref": "§2.2 ('Sigmoid output gate', following Qiu et al., 2025)" }, { "value": "Negative-eigenvalue extension of the gated delta rule: write strength β = 2σ(·) ∈ (0,2) applied to both the erase term βkkᵀS and write term βkvᵀ widens the state-transition eigenvalue range to [−1,1], letting the recurrent state flip sign and self-correct (enables parity/modular-counting state tracking)", "provenance": "stated", "ref": "§2.2 ('Negative eigenvalues'; allow_neg_eigval=True)" } ] } }, "context": { "trained_length": { "value": 1048576, "provenance": "stated", "ref": "§3.3 (Stage 4 Length Expansion 'extends training to a 1M context length'), Table 1 ('Context Length 1,048,576 (1M)')" }, "deployed_length": { "value": 1048576, "provenance": "stated", "ref": "Table 1 ('Context Length 1,048,576 (1M)'); §2.2 ('usable context window beyond 1M tokens')" }, "extension_method": { "value": "none — no positional-encoding extension needed (NoPE); the 1M window is reached by staged training, with Stage 4 (Length Expansion, 0.9T tokens) extending the context window to 1M", "provenance": "stated", "ref": "§2.2, §3.3" } }, "tokenizer": { "algorithm": { "value": "custom byte-level BPE", "provenance": "stated", "ref": "§2.1 ('a custom byte-level BPE tokenizer with a 196,608-token vocabulary')" }, "notes": { "value": "inherited unchanged from Solar Open 1; trained on a corpus oversampling Korean and target domains, with digit splitting and whitespace preservation for arithmetic and code fidelity; 4.41 bytes/token on Ko-GDPval task prompts (first among 12 tokenizers compared, ~24% above the best global model MiniMax-M3); identical vocabulary across generations is a precondition for transferring embedding/output-layer weights", "provenance": "stated", "ref": "§2.1" } }, "training": { "tokens": { "value": 12, "provenance": "stated", "ref": "§3, Figure 5 caption ('Stages 2–4 then train on about 12T tokens in total: General Pre-training (10T), Intensive Pre-training (1T), and Length Expansion (0.9T)')" }, "data_composition": { "value": "20T initially cleaned pool refined to 10T: every dataset receives an in-house quality score and rarity tracking by source/type; exact and semantic deduplication; mixture-ratio optimization via ablations — real-to-synthetic ratio 4:6, math and code each at least 15%, English share above 80%", "provenance": "stated", "ref": "§3.2" }, "curriculum": { "value": "four-stage pre-training: Stage 1 Selective Weight Transfer (2.3% of parameters from Solar Open 1), Stage 2 General Pre-training (10T tokens, broad-coverage mixture), Stage 3 Intensive Pre-training (1T tokens, higher quality threshold on the retained subset), Stage 4 Length Expansion (0.9T tokens, long-document data incl. repo-level code, extends context to 1M); four intermediate checkpoints merged into the final model (highest PLM score 0.745)", "provenance": "stated", "ref": "§3, §3.3" }, "optimizer": { "value": null, "provenance": "unknown", "note": "not disclosed (the transfer ablation mentions 'same architecture, data, and optimizer' but names none)" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "master training precision not stated (fp32 appears only as the MOPD logits-buffer format, §4.2.2)" }, "parallelism": { "value": "Context Parallelism (CP): student and teacher models are sharded by CP rank during RL/MOPD training, and teacher transport ships only the matching CP shard, skipping the CP all-gather; other parallelism dimensions (TP/PP/EP) not disclosed", "provenance": "stated", "ref": "§4.2.2 ('Under context parallelism, the teacher's CP rank r already holds exactly the shard student CP rank r needs')" }, "hardware": { "value": null, "provenance": "unknown", "note": "no GPU model named; acknowledgments credit Lablup Inc. for 'building and operating the GPU clusters and large-scale training infrastructure'" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4 ('Post-training of Solar Open 2 proceeds in four stages: (1) Supervised Fine-Tuning (SFT)...'; SFT 'targets the full range of reasoning and agent tasks')" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§4.2.1 (fully asynchronous RL with verifiable rewards; 'Retaining token-level ratios as in GRPO (Shao et al., 2024)' with trust-region clipping, stale-trajectory masking via fresh-token ratio threshold ρ, and sample dropping — GRPO-style but modified, so tagged other)" }, "reasoning_training": { "value": "Multi-domain RL strengthens core reasoning with verifiable rewards over a broad set of STEM-centered tasks; twelve domain specialists (reasoning: math/STEM/code; agents & tools: coding, general agents, officework) trained via domain SFT and domain-specific RL with verifiable rewards; SFT covers the full range of reasoning and agent tasks", "provenance": "stated", "ref": "§4, §4.2.2" }, "distillation": { "value": "Multi-Teacher On-Policy Distillation (MOPD): the student is trained on its own rollouts against twelve domain-specialist teachers; per-position distillation KL computed in closed form over the full 196,608-token vocabulary (teacher ships pre-lm_head hidden states, student rebuilds logits in 1,024-token tiles); each sample is routed to exactly one teacher, keeping teacher compute O(batch); note the selective weight transfer from Solar Open 1 is initialization, explicitly 'without distillation'", "provenance": "stated", "ref": "§4.2.2, §3.1" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "report describes only text-based reasoning, coding, tool use, and officework; no non-text modality or vision encoder appears anywhere" }, "attachment": { "value": "n/a", "provenance": "n/a", "ref": "§1", "note": "text-only; no non-text modality" } }, "efficiency": { "kv_cache": { "value": "hybrid cache: 12 softmax GQA layers keep a KV cache with 8 KV heads, while 36 linear KDA layers keep a fixed-size recurrent state and no KV cache — total cache constant in sequence length, roughly one quarter of an all-softmax stack's memory", "provenance": "derived", "note": "from attention_variant/attention_layer_pattern (Table 1 note a: 'the linear layers use 64 heads and keep no KV cache'; §2.2: 'keeping the KV cache constant in sequence length')" }, "quantization": { "value": "none disclosed", "provenance": "inferred", "note": "acknowledgments credit Nota Inc. for 'model quantization enabling efficient inference and serving' but no formats are named" }, "speculative_dedup": { "value": "none disclosed", "provenance": "inferred", "note": "no speculative decoding or multi-token prediction mentioned" }, "serving": { "value": "none disclosed", "provenance": "inferred", "note": "no serving stack or optimisations named (only the Nota Inc. quantization acknowledgment and MOPD's GPU-memory tricks for training)" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 86.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 86.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HLE (w/o tools)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 28.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "LiveCodeBench (v6)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 92.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "ArtifactsBench", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 55.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "HMMT2602", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 93.9, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME2026", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 95.7, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Multi-Challenge", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 61.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "IFBench", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 80.0, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AA-LCR", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 62.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "SWE-Bench Verified", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 70.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "Terminal Bench Hard", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 28.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "APEX-Agents", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 16.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "MCP-Atlas", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 58.2, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "τ3 (banking)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 19.6, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "GDPval-AA v2 (ELO)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 1128, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "KMMLU-Pro", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 78.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "CLIcK", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 90.7, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Ko-AIME'25", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 97.7, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HRM8K", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 92.2, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "KorMedMCQA", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 93.0, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Ko-GDPval", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 86.8, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "Korean suite Avg.", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 85.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Hybrid attention stack — three KDA linear-attention layers per one GQA softmax layer (softmax-first S-L-L-L ordering), with NoPE, a sigmoid output gate, and the negative-eigenvalue extension of the gated delta rule — reaching a 1M-token window at about one quarter of an all-softmax stack's memory and computation", "provenance": "stated", "ref": "Abstract, §2.2" }, "ref": { "value": "Abstract, §2.2", "provenance": "stated", "ref": "Abstract, §2.2" } }, { "text": { "value": "Selective weight transfer: the 5.69B-parameter shared skeleton (2.3% of the 250B total; 5.6% of Solar Open 1's 102B) is initialized from Solar Open 1 without distillation, absorbing the 128→320 expert expansion through large-scale full pre-training — claimed as the first large hybrid model pre-trained this way", "provenance": "stated", "ref": "Abstract, §3.1" }, "ref": { "value": "Abstract, §3.1", "provenance": "stated", "ref": "Abstract, §3.1" } }, { "text": { "value": "Value-per-token data curation: quality- and rarity-aware curation with mixture-ratio optimization refines a 20T pool into a 10T mixture that outperforms the Solar Open 1 recipe at equal token budget", "provenance": "stated", "ref": "Abstract, §3.2" }, "ref": { "value": "Abstract, §3.2", "provenance": "stated", "ref": "Abstract, §3.2" } }, { "text": { "value": "Twelve domain specialists consolidated into one deployable model by Multi-teacher On-Policy Distillation (MOPD), with the distillation KL computed exactly over the full 196,608-token vocabulary (no single-draw estimator) and teacher compute kept O(batch) by per-sample teacher routing", "provenance": "stated", "ref": "Abstract, §4.2.2" }, "ref": { "value": "Abstract, §4.2.2", "provenance": "stated", "ref": "Abstract, §4.2.2" } }, { "text": { "value": "A fully asynchronous RL system for 250B-scale training: stale-trajectory masking by fresh-token ratio, trust-region updates with token-level ratios, and packed teacher-pool management (one GPU-resident teacher plus twelve CPU snapshots)", "provenance": "stated", "ref": "§4.2.1, §4.2.2" }, "ref": { "value": "§4.2.1, §4.2.2", "provenance": "stated", "ref": "§4.2.1, §4.2.2" } }, { "text": { "value": "Korean-efficient tokenizer (inherited from Solar Open 1): 4.41 bytes per token on Ko-GDPval task prompts, first among 12 tokenizers compared and about 24% above the best global model (MiniMax-M3); global-model tokenizers spend 1.2–1.9× more tokens on the same Korean text", "provenance": "stated", "ref": "§1, §2.1" }, "ref": { "value": "§1, §2.1", "provenance": "stated", "ref": "§1, §2.1" } } ], "assessment": { "value": "The block is a well-executed synthesis of mechanisms already in the atlas: KDA linear attention (moonshot-kimi-k3), GQA (llama-3.1 lineage), NoPE hybrid stacking (kimi-k3), and sigmoid output gating (Qiu et al. 2025, per the report's own citation). The two genuinely differentiating choices are (1) softmax-first (S-L-L-L) ordering, explicitly opposite to Kimi Linear/Qwen3.5's L-L-L-S, and (2) the negative-eigenvalue extension (β = 2σ(·) ∈ (0,2)) that widens the gated delta rule's state-transition eigenvalues to [−1,1] — an incremental but real generalization of the KDA recurrence that Kimi's own lower-bounded-decay parameterization does not provide. At the system level, selective weight transfer across an architectural change without distillation (only 2.3% of parameters reused) and exact full-vocabulary MOPD at 250B scale are notable engineering contributions. The 250B-A15B footprint with 1M context and 3:1 linear:softmax ratio is essentially the kimi-k3 hybrid recipe at one-ninth the total size with GQA instead of MLA.", "provenance": "inferred", "note": "comparison against atlas entries (moonshot-kimi-k3, deepseek-v3, meta-llama-3.1) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "48/48 layers have MoE FFNs (320 routed + 1 shared expert, top-8), §2" }, "sequence_mixing": { "value": "hybrid", "provenance": "derived", "note": "36/48 layers (75%) linear KDA + 12/48 (25%) softmax GQA; 75% < 80% majority threshold, deliberate 3:1 per-block alternation" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred text-only)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data composition described but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "250B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a Korean sovereign frontier model ('an openly available, nationally grounded frontier model') built for long-horizon agentic tasks with broad English/Korean benchmark coverage (Abstract, §1). Secondary intents: long-context (1M) and agentic emphasis, noted in prose." } }, "prose": { "design_overview": { "value": "Solar Open 2 is a 250B-parameter sparse MoE (15B active) that scales the Solar Open 1 backbone — 48 layers, hidden 4,096, head dim 128, 64 query / 8 KV heads, 196,608-token vocab — while replacing its all-softmax GQA stack with a 3:1 linear:softmax hybrid. Every block of four layers runs the softmax layer first (S-L-L-L, opposite to Kimi Linear and Qwen3.5), then three KDA linear-attention layers, each paired with an MoE FFN (320 routed experts, top-8, one shared expert, SiLU, per-expert width 1,280). Three extensions define the design: NoPE — no positional encoding anywhere, so the linear layers' sequential state carries all token order and the model extrapolates in principle without bound; a sigmoid output gate on the softmax layers that adds query-dependent sparsity and suppresses attention-sink collapse; and the negative-eigenvalue extension of the gated delta rule (β = 2σ(·) ∈ (0,2)), which widens the state-transition eigenvalues to [−1,1] so the recurrent state can erase and invert, not merely decay — restoring provable state-tracking (parity, modular counting) that [0,1]-eigenvalue cores lack. The 1M context is reached by staged training: 10T general pre-training, 1T intensive, then 0.9T length expansion, with four checkpoints merged. Training efficiency is attacked from two sides: a selective weight transfer that reuses only the 5.69B-parameter shape-compatible skeleton from Solar Open 1 (no distillation), and a value-per-token data recipe (20T pool → 10T, 4:6 real:synthetic). Post-training runs SFT → multi-domain RL → twelve domain specialists → Multi-teacher On-Policy Distillation, folding the specialists into one model with exact full-vocabulary KL on the student's own rollouts.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "KDA (Kimi Delta Attention) — gated delta-rule linear attention with fixed-size recurrent state for the 36 linear layers", "provenance": "stated", "ref": "§2.2 (cites Yang et al., 2025; Kimi Team, 2025)" }, "origin": { "value": "Kimi (Kimi Linear / Kimi K3 family)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Hybrid attention stack interleaving linear and softmax layers (following Kimi Linear, Qwen3.5, and Nemotron 3)", "provenance": "stated", "ref": "§2.2 (cites Kimi Team 2025; Qwen Team 2026; NVIDIA 2025)" }, "origin": { "value": "Kimi Linear / Qwen3.5 / Nemotron 3", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "GQA (grouped-query attention) for the softmax layers", "provenance": "stated", "ref": "§2.2 (cites Ainslie et al., 2023)" }, "origin": { "value": "GQA (Ainslie et al.)", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "Elementwise sigmoid output gate on softmax attention outputs", "provenance": "stated", "ref": "§2.2 ('Following Qiu et al. (2025)')" }, "origin": { "value": "Sigmoid-attention line of work (Qiu et al., not in atlas)", "provenance": "stated", "ref": "§2.2" } } ], "influence_out": [], "notable_omissions": { "value": [ "No optimizer, learning-rate schedule, batch size, or training precision disclosed", "No training hardware or FLOPs/GPU-hours disclosed (Lablup's GPU clusters credited but never specified)", "Router function and load-balancing mechanism not described (only the top-8 count)", "Normalization layer type not named (only 'pre-norm residuals')", "Quantization formats not disclosed (Nota Inc. credited only in acknowledgments)", "No license named for the weights release", "No embedding-tying statement", "SFT and Multi-domain RL data compositions explicitly 'outside the scope of this report' (§4)", "Checkpoint-merging method (Stage 4) not detailed", "Per-stage data-mixture proportions not disclosed (only the overall recipe)", "MOPD hyperparameters (ρ threshold, trust-region bounds, G) not quantified" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "upstage-solar-open-2.json" }, { "slug": { "value": "weibo-vibethinker-3b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2606.16140", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/weibo-vibethinker-3b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "VibeThinker", "provenance": "stated", "ref": "Title; §1 ('Extending the technical lineage of the VibeThinker series'); prior work VibeThinker-1.5B [42]" }, "variants": [ { "value": "VibeThinker-3B (3B dense)", "provenance": "stated", "ref": "Abstract; §1" }, { "value": "VibeThinker-1.5B (1.5B, prior work)", "provenance": "stated", "ref": "§1, §2 (previous work [42])" } ], "org": { "value": "Weibo", "provenance": "inferred", "note": "per task brief and slug; the report text itself does not name the organisation (author byline not present in the extracted text; VibeThinker-1.5B [42] is by S. Xu, Y. Zhou, W. Wang, J. Min, Z. Yin, Y. Dai, S. Liu, L. Pang, Y. Chen, J. Zhang)" }, "release_date": { "value": "2026-06-15", "provenance": "stated", "ref": "arXiv header line ('arXiv:2606.16140v1 [cs.AI] 15 Jun 2026')" }, "report_url": { "value": "https://arxiv.org/abs/2606.16140", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "arXiv header prints 'License: CC Zero' — that is the paper's arXiv license, not a model-weights license; no weights license is stated" }, "open_weights": { "value": null, "provenance": "unknown", "note": "report never states whether model weights are released (no HuggingFace/GitHub link, no release statement)" } }, "scale": { "total_params": { "value": 3, "provenance": "stated", "ref": "Abstract ('a compact dense model with 3B parameters'); §4 ('comprising only 3 billion parameters')" }, "active_params_per_token": { "value": 3, "provenance": "derived", "note": "dense model: active params = total params" }, "layers": { "value": null, "provenance": "unknown", "note": "report never describes the block; base model Qwen2.5-Coder-3B internals are not imported per schema (config-file knowledge is secondary)" }, "hidden_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_inner_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "ffn_ratio": { "value": null, "provenance": "unknown", "note": "not derivable: hidden dim and FFN dim undisclosed" }, "attention_heads_q": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "attention_heads_kv": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "head_dim": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "variants": [] }, "block": { "block_type": { "value": "dense", "provenance": "stated", "ref": "Abstract ('a compact dense model with 3B parameters'); §2 ('Qwen2.5-Coder-3B base, a compact 3B dense foundation model')" }, "moe": { "expert_count": { "value": null, "provenance": "n/a", "ref": "Abstract (dense model)" }, "experts_per_token": { "value": null, "provenance": "n/a", "ref": "Abstract (dense model)" }, "shared_experts": { "value": null, "provenance": "n/a", "ref": "Abstract (dense model)" }, "routing": { "value": null, "provenance": "n/a", "ref": "Abstract (dense model)" }, "load_balancing": { "value": null, "provenance": "n/a", "ref": "Abstract (dense model)" }, "expert_granularity": { "value": null, "provenance": "n/a", "ref": "Abstract (dense model)" } }, "attention_variant": { "value": null, "provenance": "unknown", "note": "report never describes the attention mechanism; base-model internals not imported" }, "attention_layer_pattern": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: report describes no learned attention over prior layer outputs (report covers post-training only)" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "extension": { "method": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "factor": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "target_length": { "value": null, "provenance": "unknown", "note": "the 64K figure in §2.2.2 is the RL training window, not a positional-encoding extension target" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "placement": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "qk_norm": { "value": null, "provenance": "unknown", "note": "not disclosed in report; no evidence either way" } }, "activation": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: report describes no designed sink mechanism (the report covers post-training only and never discusses the block)" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described anywhere" }, "other": [] } }, "context": { "trained_length": { "value": 65536, "provenance": "stated", "ref": "§2.2.2 ('we directly conduct RL with a single 64K long-context window')", "note": "this is the disclosed RL training window of the post-trained model, not a base-model pretraining length (pretraining is outside this report)" }, "deployed_length": { "value": null, "provenance": "unknown", "note": "no deployed/served context length stated (evaluation imposes no output cap beyond the model's maximum generation length, §3.1)" }, "extension_method": { "value": null, "provenance": "unknown", "note": "report does not discuss context extension; it explicitly rejects progressive window expansion for RL (single 64K window, §2.2.2)" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "not disclosed in report" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "no pretraining performed in this work (post-training on Qwen2.5-Coder-3B base); no post-training token counts disclosed" }, "data_composition": { "value": "SFT: multi-domain mixed dataset covering math, code, STEM reasoning, general chat, and instruction following; query synthesis/expansion from high-confidence seeds (explicit final answers, unit tests), multi-path teacher distillation with majority-voting pseudo-labels; quality control via n-gram filtering (incl. benchmark-contamination removal), LLM query-quality filtering, and trace-correctness filtering (answer verification, code sandbox, LLM majority voting). RL: math (final-answer verification), code (sandbox execution + test cases), STEM (answer matching + option verification), all under strict decontamination; Instruct RL: format-sensitive prompts, long-context instructions, general alignment examples", "provenance": "stated", "ref": "§2.1.1, §2.2.2, §2.4" }, "curriculum": { "value": "two-stage curriculum SFT (stage 1: broad coverage, 5 epochs, full filtered dataset, sequence packing; stage 2: hard long-horizon subset, 2 epochs, traces ≥5K tokens, easy problems filtered via VibeThinker-1.5B rollouts with error rate <0.75); sequential multi-domain RL: Math RL → Code RL → STEM RL at a single 64K window; Long2Short Math RL (accuracy-first MGPO, then token-efficiency reward redistribution among correct trajectories, λ=0.2); offline self-distillation; Instruct RL", "provenance": "stated", "ref": "§2.1.2, §2.2.2, §2.3, §2.4" }, "optimizer": { "value": null, "provenance": "unknown", "note": "optimizer not named (only learning-rate schedule and batch size disclosed, §2.1.2)" }, "lr_schedule": { "value": "cosine annealing from 5×10⁻⁵ decaying to 8×10⁻⁸ with 5% linear warmup (both SFT stages)", "provenance": "stated", "ref": "§2.1.2" }, "batch_schedule": { "value": "global batch size 128 (both SFT stages)", "provenance": "stated", "ref": "§2.1.2" }, "precision": { "value": null, "provenance": "unknown", "note": "training precision not disclosed" }, "parallelism": { "value": null, "provenance": "unknown", "note": "not disclosed (rollout engine noted only for training-inference probability mismatch, §2.2.1)" }, "hardware": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "not disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§2.1 (two-stage curriculum SFT with Diversity-Exploring Distillation and domain-specialist checkpoint merging)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§2.2.1, §2.4", "note": "MGPO (MaxEnt-Guided Policy Optimization): a GRPO-style clipped objective with maximum-entropy prompt weighting w(q)=exp(−γ·D_ME(p(q)‖0.5)) applied to group-relative advantages, run fully on-policy; Instruct RL uses rule-based validators plus rubric-based reward models under the same on-policy framework" }, "reasoning_training": { "value": "multi-domain verifiable reasoning RL (Math → Code → STEM) with MGPO at a single 64K long-context window (progressive window expansion rejected); Long2Short Math RL redistributing rewards toward shorter correct trajectories; CLR (Claim-Level Reliability Assessment) claim-level test-time scaling — K=32 candidate trajectories, M=5 decision-relevant claims extracted per trajectory, self-verification verdicts mapped through a nonlinear reliability score (r_k = (mean verdict)^M), reliability-weighted answer aggregation — evaluated over 8 independent runs; curriculum CoT SFT with a 5K-token trace threshold", "provenance": "stated", "ref": "§2.2.2, §3.1" }, "distillation": { "value": "offline self-distillation: verified high-learning-potential trajectories (S_LP = length-normalized student NLL, bucketed per domain/length) from the model's own Math/Code/STEM RL checkpoints are distilled back into a unified student via SFT; note: the SFT-stage 'multi-path distillation' uses external strong-teacher models to generate training data, which the schema treats as teacher-generated data (see prose), not distillation", "provenance": "stated", "ref": "§2.3, §2.1.1" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: all training stages and benchmarks described are textual (math, code, STEM, instruction); no non-text modality appears" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1–§3 (text-only model)" } }, "efficiency": { "kv_cache": { "value": null, "provenance": "unknown", "note": "attention variant undisclosed, so no KV-cache consequence can be derived; report does not discuss KV cache" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "§2–§3", "note": "no quantization formats disclosed (vLLM is used for evaluation inference, §3.1)" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "§2–§3", "note": "no speculative decoding or deduplication mechanisms disclosed" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "§3.1", "note": "vLLM appears only as the evaluation inference backend; no serving optimizations disclosed" } }, "evaluation": { "benchmarks": [ { "name": { "value": "AIME26 (Pass@1, 64 samples)", "provenance": "stated", "ref": "Table 1; §3.1" }, "value": { "value": 94.3, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "AIME25 (Pass@1, 64 samples)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 91.4, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "HMMT25 (Pass@1, 64 samples)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 89.3, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "BruMO25 (Pass@1, 64 samples)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 93.8, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "IMO-AnswerBench (Pass@1, 16 samples)", "provenance": "stated", "ref": "Table 1; §3.1" }, "value": { "value": 76.4, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "LiveCodeBench v6 (Pass@1)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 80.2, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "OJBench (Pass@1)", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 38.6, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 70.2, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "IFEval", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 93.4, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "IFBench", "provenance": "stated", "ref": "Table 1" }, "value": { "value": 74.5, "provenance": "stated", "ref": "Table 1" }, "ref": { "value": "Table 1", "provenance": "stated", "ref": "Table 1" } }, { "name": { "value": "LeetCode weekly & biweekly contests (Apr 25–May 31 2026, acceptance rate)", "provenance": "stated", "ref": "Table 3; §3.2" }, "value": { "value": "96.1 (123/128)", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME26 + CLR (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 97.1, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "AIME25 + CLR (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 96.7, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "HMMT25 + CLR (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 95.4, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "BruMO25 + CLR (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 99.2, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "IMO-AnswerBench + CLR (Pass@1)", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 80.6, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } }, { "name": { "value": "GPQA-Diamond + CLR", "provenance": "stated", "ref": "Table 2" }, "value": { "value": 72.9, "provenance": "stated", "ref": "Table 2" }, "ref": { "value": "Table 2", "provenance": "stated", "ref": "Table 2" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Frontier-level verifiable reasoning at a strict 3B scale: 94.3 on AIME26 and 80.2 Pass@1 on LiveCodeBench v6, matching or exceeding flagship models orders of magnitude larger (DeepSeek V3.2, GLM-5, Gemini 3 Pro, Kimi K2.5)", "provenance": "stated", "ref": "Abstract, §1, Table 2" }, "ref": { "value": "Abstract, §1, Table 2", "provenance": "stated", "ref": "Abstract, §1, Table 2" } }, { "text": { "value": "Parametric Compression-Coverage Hypothesis: verifiable reasoning is a parameter-dense capability compressible into a compact reasoning core, while knowledge and general-purpose competence are parameter-expansive coverage problems", "provenance": "stated", "ref": "§1, §4" }, "ref": { "value": "§1, §4", "provenance": "stated", "ref": "§1, §4" } }, { "text": { "value": "Reasoning-Knowledge Decoupling Paradigm: small models are a complementary research trajectory to parameter scaling rather than a deployment compromise", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } }, { "text": { "value": "A systematically upgraded Spectrum-to-Signal post-training pipeline: two-stage curriculum SFT, multi-domain reasoning RL (Math → Code → STEM), Long2Short Math RL for token-efficiency, offline self-distillation with learning-potential filtering, and Instruct RL", "provenance": "stated", "ref": "§2" }, "ref": { "value": "§2", "provenance": "stated", "ref": "§2" } }, { "text": { "value": "CLR (Claim-Level Reliability Assessment): a claim-level test-time scaling strategy that extracts and self-verifies decision-relevant claims, improving Pass@1 on answer-verifiable math (AIME26 94.3 → 97.1) at lower token cost than trace-level verification", "provenance": "stated", "ref": "§3.1" }, "ref": { "value": "§3.1", "provenance": "stated", "ref": "§3.1" } }, { "text": { "value": "Out-of-distribution generalization: 96.1% first-attempt acceptance (123/128) on recent unseen LeetCode contests, above GPT-5.2, Kimi K2.5, and Qwen3-Max under the same protocol", "provenance": "stated", "ref": "§1, §3.2, Table 3" }, "ref": { "value": "§1, §3.2, Table 3", "provenance": "stated", "ref": "§1, §3.2, Table 3" } } ], "assessment": { "value": "This is a post-training recipe report on a fixed Qwen2.5-Coder-3B base: the block is entirely undisclosed, so there is no architectural novelty relative to atlas entries. The post-training stack is a GRPO-family variant (MGPO adds maximum-entropy prompt weighting to the clipped objective — a modest, well-motivated twist on deepseek-r1's verifiable-RL recipe), plus Long2Short length-reward redistribution (echoing length-bias control seen in the deepseek-r1/post-training literature), offline self-distillation from own checkpoints (similar to deepseek-r1's R1-distill consolidation), and CLR, a claim-level test-time scaling scheme that is a genuine contribution relative to trace-level self-verification (deepseek-r1's 'verify' mode). The headline claims (matching 671B-class models on AIME26) are internally consistent but rest on 2026 competition sets and majority of evaluation is Pass@1 over 64 samples with LLM-as-judge assistance on complex answers; the knowledge gap (GPQA-D 70.2 vs 80+ for frontier peers) is honestly reported and matches the compression-coverage thesis. Nothing here contradicts atlas peers; the report's own framing admits reasoning-knowledge decoupling rather than general frontier parity.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-r1, microsoft-phi-4, allenai-olmo-2-7b, microsoft-phi-3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "dense", "provenance": "stated", "ref": "Abstract ('a compact dense model with 3B parameters')" }, "sequence_mixing": { "value": "full-attention", "provenance": "inferred", "note": "report never describes the block; base model Qwen2.5-Coder-3B (§2) is a standard dense full-attention transformer per its public configuration (secondary knowledge). The report contains no hint of SSM/efficient-attention/hybrid mixing." }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "undisclosed", "provenance": "derived", "note": "open_weights not stated in the report; data composition described but no release of data or weights mentioned" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "3B: 1B ≤ 3B < 10B" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "framed entirely around verifiable reasoning ('Exploring the Frontier of Verifiable Reasoning in Small Language Models', title); math/code/STEM reasoning with CLR test-time scaling; instruction controllability preserved as a constraint, not a goal" } }, "prose": { "design_overview": { "value": "VibeThinker-3B is a post-training-only project: a fixed 3B dense base (Qwen2.5-Coder-3B, named but never described internally) is driven to frontier-level verifiable-reasoning performance through a five-stage pipeline built on the authors' Spectrum-to-Signal Principle. Stage 1 is a two-stage curriculum SFT: broad multi-domain coverage (math, code, STEM, chat, instruction following) over 5 epochs, then 2 epochs on a hard subset of traces ≥5K tokens filtered by VibeThinker-1.5B rollouts, with Diversity-Exploring Distillation merging domain-specialist checkpoints at the parameter level to preserve a wide solution 'spectrum'. Stage 2 runs MGPO — a GRPO-style clipped policy objective reweighting prompts by their proximity to the maximum-entropy point p=0.5, keeping training at the capability boundary — sequentially over Math, Code, and STEM RL at a single 64K context window (progressive window expansion is rejected as harmful at this scale), followed by Long2Short Math RL, which redistributes reward among correct trajectories toward shorter ones to compress reasoning length. Stage 3 is offline self-distillation: verified trajectories from the Math/Code/STEM checkpoints, selected by a learning-potential score, are distilled back into one student. Stage 4 is Instruct RL with rule-based and rubric rewards to restore strict instruction controllability. At inference, CLR scales test-time compute at the claim level: 32 trajectories, five decision-relevant claims each, self-verified and aggregated by a reliability-weighted vote. The report's thesis — the Parametric Compression-Coverage Hypothesis — is that verifiable reasoning compresses into a small 'reasoning core' while knowledge needs broad parameter coverage, which the GPQA-Diamond gap (70.2) versus frontier peers illustrates.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Spectrum-to-Signal Principle (SSP): SFT builds a diverse solution spectrum, RL amplifies the signal", "provenance": "stated", "ref": "§2 (introduced in VibeThinker-1.5B [42])" }, "origin": { "value": "VibeThinker-1.5B", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "MaxEnt-Guided Policy Optimization (MGPO): GRPO-style clipped objective with maximum-entropy prompt weighting", "provenance": "stated", "ref": "§2.2.1 (introduced in VibeThinker-1.5B [42])" }, "origin": { "value": "VibeThinker-1.5B", "provenance": "stated", "ref": "§2.2.1" } }, { "mechanism": { "value": "GRPO-style clipped policy objective with group-relative advantages", "provenance": "stated", "ref": "§2.2.1 (Eq. 3; cites DeepSeekMath GRPO [32])" }, "origin": { "value": "DeepSeekMath (DeepSeek)", "provenance": "stated", "ref": "§2.2.1" } }, { "mechanism": { "value": "Base model: Qwen2.5-Coder-3B dense foundation model", "provenance": "stated", "ref": "§2" }, "origin": { "value": "Qwen2.5-Coder-3B (Alibaba)", "provenance": "stated", "ref": "§2" } }, { "mechanism": { "value": "Progressive context-window expansion RL considered and rejected in favor of a single 64K window", "provenance": "stated", "ref": "§2.2.2 (cites DeepScaleR [21])" }, "origin": { "value": "DeepScaleR", "provenance": "stated", "ref": "§2.2.2" } }, { "mechanism": { "value": "Diversity-Exploring Distillation: domain-specialist checkpoint merging", "provenance": "stated", "ref": "§2.1.2 (from VibeThinker-1.5B [42])" }, "origin": { "value": "VibeThinker-1.5B", "provenance": "stated", "ref": "§2.1.2" } }, { "mechanism": { "value": "On-policy RL stabilization against rollout-training probability mismatch", "provenance": "stated", "ref": "§2.2.1 (cites Mismatch Praxis [13], Yao et al. [43])" }, "origin": { "value": "Mismatch Praxis / efficient-RL-framework analyses (not in atlas)", "provenance": "stated", "ref": "§2.2.1" } } ], "influence_out": [], "notable_omissions": { "value": [ "Model architecture entirely undisclosed: layers, hidden dim, heads, activation, position encoding, vocab, normalization — none appear (post-training-only report)", "No pretraining information: tokens, data, compute, hardware (base model is external)", "Model weights release not stated; no HuggingFace/GitHub repository or model card linked", "Organisation not named anywhere in the report text", "No license for the model weights (arXiv 'CC Zero' header is the paper license only)", "RL hyperparameters largely undisclosed (γ, ε, rollout counts, reward-model details)", "Optimizer, training precision, parallelism, and hardware not disclosed", "No error bars or per-problem breakdowns for most benchmark numbers; CLR results are means over 8 runs only", "No safety/robustness evaluation beyond instruction-following (IFEval/IFBench)", "Knowledge benchmarks limited to GPQA-Diamond; no MMLU/SimpleQA-style coverage to quantify the acknowledged knowledge gap", "VibeThinker-1.5B comparison numbers not tabulated in this report (only referenced)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "weibo-vibethinker-3b.json" }, { "slug": { "value": "xiaomi-mimo-v2-flash-309b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2601.02780", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/xiaomi-mimo-v2-flash-309b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "MiMo-V2", "provenance": "stated", "ref": "Title (MiMo-V2-Flash Technical Report); Table 9 header splits 'MiMo-V2' / 'Flash'" }, "variants": [ { "value": "MiMo-V2-Flash", "provenance": "stated", "ref": "Title, Abstract" }, { "value": "MiMo-V2-Flash-Base", "provenance": "stated", "ref": "§1 (pretrained base model evaluated in §3.3)" } ], "org": { "value": "Xiaomi (LLM-Core)", "provenance": "stated", "ref": "Byline 'LLM-Core Xiaomi'" }, "release_date": { "value": "2026-01-06", "provenance": "inferred", "note": "Report text is undated; arXiv v1 submission date 2026-01-06 per export.arxiv.org API metadata (arXiv:2601.02780)" }, "report_url": { "value": "https://arxiv.org/abs/2601.02780", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "report never names a license; weights linked via GitHub repo" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract: 'We open-source both the model weights and the three-layer MTP weights'; §1 (github.com/XiaomiMiMo/MiMo-V2-Flash)" } }, "scale": { "total_params": { "value": 309, "provenance": "stated", "ref": "Abstract, §3.2, Table 5" }, "active_params_per_token": { "value": 15, "provenance": "stated", "ref": "Abstract, §3.2, Table 5 (#Activated Params 15B)" }, "layers": { "value": 48, "provenance": "stated", "ref": "Table 1 (Layers Total/SWA/GA = 48/39/9), §3.2" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "§3.2 ('The hidden dimension is set to 4096')" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "ref": "§3.2 ('an intermediate hidden dimension of 2048 for each expert'); dense first-layer FFN intermediate is 16384" }, "ffn_ratio": { "value": 0.5, "provenance": "derived", "note": "per-expert ratio: 2048 / 4096 = 0.5 (dense first-layer FFN ratio is 16384 / 4096 = 4.0)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 1 (SWA Heads (Q/KV) 64/8; GA Heads (Q/KV) 64/4)" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1", "note": "SWA layers have 8 KV heads; the 9 GA layers have 4 KV heads (both GQA)" }, "head_dim": { "value": 192, "provenance": "stated", "ref": "Table 1 (Head Dimensions (QK/V) 192/128 for both SWA and GA)", "note": "report states QK head dim 192 and V head dim 128; the naive hidden/heads quotient (4096/64 = 64) does not match the stated per-head projection dims" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "vocabulary size never disclosed; tokenizer not described" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "embedding/output head tying not disclosed (BF16 kept 'for the embedding and output head parameters' but tying not stated)" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "47/48 layers use routed MoE FFNs; only the very first Transformer block has a dense FFN (Table 1, §2.1)" }, "moe": { "expert_count": { "value": 256, "provenance": "stated", "ref": "§2.1 ('Each MoE layer comprises 256 experts in total'), Table 1" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "§2.1 ('8 activated per token'), Table 1 (Experts (Total/Activated) 256/8)" }, "shared_experts": { "value": 0, "provenance": "stated", "ref": "§2.1 ('and contains no shared experts')" }, "routing": { "value": null, "provenance": "unknown", "note": "router function never described (only 'MoE sequence auxiliary loss' and 'expert bias update factor' hyperparameters appear, §3.2)" }, "load_balancing": { "value": "sequence-wise MoE auxiliary loss (coefficient 1.0e-5 all stages) plus per-expert bias updates (bias update factor 0.001 in Stages 1-2, 1.0e-5 in Stage 3; 1.0e-4 during SFT)", "provenance": "stated", "ref": "§3.2, §4.2" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "granularity not described; 256 experts / 8 active = 32x sparsity with per-expert width 2048 (0.5x hidden), no shared experts" } }, "attention_variant": { "value": "hybrid", "provenance": "stated", "ref": "§2.1 (interleaved Sliding Window Attention and Global Attention, both GQA)" }, "attention_layer_pattern": { "value": "8 hybrid blocks of 5 SWA + 1 GA; the very first Transformer block is GA with a dense FFN; totals 39 SWA + 9 GA layers; SWA window = 128 tokens", "provenance": "stated", "ref": "§2.1, Table 1" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard Transformer residual backbone (§2.1), no learned attention over prior layer outputs described" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1 ('Rotary Positional Embedding (RoPE) is partially applied to the first 64 dimensions of query and key')" }, "rope_base": { "value": 5000000, "provenance": "stated", "ref": "§3.2 (Long Context Extension)", "note": "deployed GA base 5,000,000 (Stage 3); SWA base 10,000 throughout; initial Stage-1 GA base was 640,000" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§2.1 (RoPE applied to first 64 QK dimensions)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "context extension via Stage-3 training at 262,144 with GA RoPE base raised 640,000 -> 5,000,000 (SWA base unchanged at 10,000); no YaRN/NTK/PI named", "ref": "§3.2, §3.1 Stage 3" }, "factor": { "value": 7.8125, "provenance": "derived", "note": "5,000,000 / 640,000 = 7.8125 (both bases stated in §3.2)" }, "target_length": { "value": 262144, "provenance": "stated", "ref": "§3.1 Stage 3 ('extend the model's context window to 256K tokens')" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "normalization type never named in the report" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: no QK-normalisation described on the global-attention layers" } }, "activation": { "value": null, "provenance": "unknown", "note": "FFN activation function never named (no SwiGLU/GELU mention anywhere)" }, "stability": { "attention_sinks": { "value": true, "provenance": "stated", "ref": "§2.2 (learnable attention sink bias s_sink added to the softmax denominator per head, following gpt-oss)" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no logit softcapping described" }, "other": [ { "value": "FP8 mixed-precision configuration for stability: BF16 retained for attention output projections and embedding/output head parameters, FP32 for the MoE router (following DeepSeek-V3)", "provenance": "stated", "ref": "§2.1" }, { "value": "num-zeros (parameters with zero gradients) tracked as an MoE SFT stability metric; expert bias update rate and AdamW epsilon tuned against it", "provenance": "stated", "ref": "§4.2" }, { "value": "Rollout Routing Replay (R3): RL trains with the same routed experts as rollout to avoid precision-driven routing mismatch", "provenance": "stated", "ref": "§4.6.1" } ] } }, "context": { "trained_length": { "value": 262144, "provenance": "stated", "ref": "§3.1 Stage 3 (context extended to 256K in the 26-27T range)" }, "deployed_length": { "value": 262144, "provenance": "stated", "ref": "Abstract, §1 ('native 32k context length and subsequently extended to 256k')" }, "extension_method": { "value": "Stage-3 context-extension training (26-27T) at 262,144 with long-range data upsampled; GA RoPE base raised 640,000 -> 5,000,000", "provenance": "stated", "ref": "§3.1, §3.2" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer algorithm not described" }, "notes": { "value": null, "provenance": "unknown", "note": "no tokenizer details disclosed" } }, "training": { "tokens": { "value": 27, "provenance": "stated", "ref": "Abstract, §3 ('27 trillion tokens')" }, "data_composition": { "value": "diverse high-quality corpus: public web content, books, academic papers, code, mathematics, and broader STEM; deliberate shift toward long-range dependencies (long-form web documents, repository-level code, pull requests, issues, commit histories); ~5% synthetic reasoning data added in Stage 2; pipeline largely follows MiMo-7B", "provenance": "stated", "ref": "§3, §3.1" }, "curriculum": { "value": "three stages: Stage 1 (0-22T) general corpus at 32K context; Stage 2 (22-26T) code upsampling + ~5% synthetic reasoning data; Stage 3 (26-27T) context extension to 256K with long-range data upsampled", "provenance": "stated", "ref": "§3.1" }, "optimizer": { "value": "AdamW with beta1=0.9, beta2=0.95, weight decay 0.1, gradient clipping max norm 1.0", "provenance": "stated", "ref": "§3.2" }, "lr_schedule": { "value": "Stage 1: linear warmup 0 -> 3.2e-4 over 50B tokens, constant 3.2e-4 for 12T, cosine decay to 1.0e-4 over 10T; Stage 2: cosine 1.0e-4 -> 3.0e-5 over 4T; Stage 3: cosine 3.0e-5 -> 1.0e-5", "provenance": "stated", "ref": "§3.2" }, "batch_schedule": { "value": "batch size warms up linearly to 2048 over the first 500B tokens and stays constant through Stages 1-2; fixed batch size 256 in Stage 3; SFT batch size 128", "provenance": "stated", "ref": "§3.2, §4.2" }, "precision": { "value": "FP8 mixed precision (DeepSeek-V3-style): BF16 for attention output projections and embedding/output head, FP32 for MoE router; FP8 used for both training and inference", "provenance": "stated", "ref": "§2.1, §4.6" }, "parallelism": { "value": null, "provenance": "unknown", "note": "training parallelism (TP/PP/EP/CP) not disclosed; RL infra uses SGLang (inference) + Megatron-LM (training)" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§4.1 Stage 1, §4.2 (millions of samples across conversation, reasoning, coding, agent tasks; thinking and non-thinking modes)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§4.4 ('we combine the advantages of MOPD with other types of advantages, such as those computed using Outcome Reward Models (ORMs), including GRPO [shao2024deepseekmath]')", "note": "GRPO-style ORM advantages are one component; the headline mechanism is MOPD (reverse-KL on-policy distillation rewards, Eq. 5-9)" }, "reasoning_training": { "value": "large-scale RL: non-agentic RL on verifiable domains (programmatic verifiers + LLM judge, rubric-based judge for subjective qualities); agentic RL across code (120K environments), terminal (~30K), web development, search (150K), function-calling (50K) agents; all consolidated via MOPD distillation", "provenance": "stated", "ref": "§4.3, Table 8" }, "distillation": { "value": "Multi-Teacher On-Policy Distillation (MOPD): student samples from its own distribution and receives dense token-level reverse-KL rewards from domain-specialized teacher policies (RL or SFT trained), combined with ORM advantages; training-inference importance sampling with token discarding (IcePop2025)", "provenance": "stated", "ref": "§4.1 Stage 3, §4.4 (Eq. 5-9)" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "no vision or other modality described anywhere; all evaluation tasks are text-based" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type (text-only)" } }, "efficiency": { "kv_cache": { "value": "hybrid: 39/48 layers are SWA with a 128-token window so their KV cache is bounded by the window; 9 GA layers keep full cache; report claims nearly 6x reduction in KV-cache storage and attention computation for long contexts", "provenance": "stated", "ref": "§1, §2.1" }, "quantization": { "value": "FP8 (used for both training and inference)", "provenance": "stated", "ref": "§4.6 ('We adopt FP8 for both training and inference')" }, "speculative_dedup": { "value": "self-speculative decoding with MTP as native draft model: single MTP head during pre-training, replicated to K=3 jointly-trained heads in post-training; up to 3.6 average acceptance length and 2.6x decoding speedup", "provenance": "stated", "ref": "§2.3, Abstract, §5" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report is silent on serving optimizations beyond MTP-based speculative decoding" } }, "evaluation": { "benchmarks": [ { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 84.9, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 84.3, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "HLE (no tools)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 22.1, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "AIME 2025", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 94.1, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "HMMT Feb. 2025", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 84.4, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "LiveCodeBench-v6", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 85.1, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "Arena-Hard (Hard Prompt)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 54.1, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "Arena-Hard (Creative Writing)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 86.2, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "LongBench V2", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 60.6, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "MRCR", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 45.7, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "SWE-Bench Verified", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 73.4, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "SWE-Bench Multilingual", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 71.7, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "Terminal-Bench Hard", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 30.5, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "Terminal Bench 2.0", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 38.5, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 45.4, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "BrowseComp (w/ Context Manage)", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 58.3, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "tau2-Bench", "provenance": "stated", "ref": "Table 9" }, "value": { "value": 80.3, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "NIAH-Multi (256K)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 96.7, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "GSM-Infinite Hard (128K)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 29.0, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MMLU (Base)", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 86.7, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Hybrid SWA architecture: 5:1 local:global interleaving with a 128-token window and learnable attention sink bias, yielding ~6x KV-cache/compute reduction while matching or surpassing all-global baselines on general, long-context, and reasoning benchmarks", "provenance": "stated", "ref": "§1, §2.2" }, "ref": { "value": "§1, §2.2", "provenance": "stated", "ref": "§1, §2.2" } }, { "text": { "value": "Multi-Teacher On-Policy Distillation (MOPD): a three-stage post-training paradigm (SFT, domain-specialized RL/SFT teachers, on-policy distillation with dense token-level rewards plus ORM advantages) that lets the student master peak teacher capabilities without see-saw trade-offs", "provenance": "stated", "ref": "§4.1" }, "ref": { "value": "§4.1", "provenance": "stated", "ref": "§4.1" } }, { "text": { "value": "Lightweight MTP repurposed as a native draft model for self-speculative decoding (up to 3.6 acceptance length, 2.6x decoding speedup) and for enabling small-batch on-policy RL rollouts", "provenance": "stated", "ref": "§2.3, §5" }, "ref": { "value": "§2.3, §5", "provenance": "stated", "ref": "§2.3, §5" } }, { "text": { "value": "Competitive with DeepSeek-V3.2 and Kimi-K2 using 1/2-1/3 of their total parameters; leading open-source software-engineering results (73.4% SWE-Bench Verified, 71.7% SWE-Bench Multilingual)", "provenance": "stated", "ref": "Abstract, §4.5.2" }, "ref": { "value": "Abstract, §4.5.2", "provenance": "stated", "ref": "Abstract, §4.5.2" } }, { "text": { "value": "Long-context robustness: near-100% NIAH retrieval success from 32K to 256K and superior GSM-Infinite retention vs sparse-attention peers; surpasses larger full-attention models on LongBench V2 and MRCR", "provenance": "stated", "ref": "§1, §3.3.2" }, "ref": { "value": "§1, §3.3.2", "provenance": "stated", "ref": "§1, §3.3.2" } }, { "text": { "value": "RL infrastructure for ultra-scaled agentic training: Rollout Routing Replay (R3), fine-grained Data Scheduler with partial rollout, and Toolbox/Tool Manager for 300K+ agentic tasks", "provenance": "stated", "ref": "§4.6" }, "ref": { "value": "§4.6", "provenance": "stated", "ref": "§4.6" } }, { "text": { "value": "Open-sourced model weights and the three-layer MTP weights", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } } ], "assessment": { "value": "The block is a competent synthesis of known mechanisms rather than a new mixing family: the 5:1 SWA/GA schedule extends the Gemma-style local/global interleave, the learnable attention sink bias is taken from OpenAI gpt-oss (in atlas), and FP8 mixed precision + MTP + GRPO-style ORMs follow the DeepSeek-V3/R1 playbook. The genuinely distinctive contributions are (a) the aggressive 128-token window at 5:1 validated with sink bias, and (b) MOPD, a named multi-teacher on-policy distillation paradigm — the same acronym and concept later appear in Moonshot Kimi K3 (arXiv 2607.24653, in atlas), which postdates this report; MiMo's January 2026 publication predates it, so the atlas now holds two independent MOPD claims. The 'rivals DeepSeek-V3.2/Kimi-K2 at 1/2-1/3 parameters' claim is internal and not independently verifiable. Report is unusually silent on normalization, activation function, router function, vocab size, and training hardware — architecture sections that peers almost always fill.", "provenance": "inferred", "note": "comparison against atlas entries (openai-gpt-oss, deepseek-v3, moonshot-kimi-k3, google-gemma-3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "47/48 layers routed MoE (1 dense layer, Table 1 / §2.1)" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "39 SWA + 9 GA layers; SWA share 39/48 = 81.25% >= 80% majority threshold (Table 1)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no multimodal content in report)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (weights + MTP weights released); data composition described but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "309B >= 100B" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "Abstract: 'designed for fast, strong reasoning and agentic capabilities'; post-training is dominated by large-scale RL for reasoning and agents. Secondary intents: long-context (256K) and fast inference (Flash), noted in prose." } }, "prose": { "design_overview": { "value": "MiMo-V2-Flash is Xiaomi's 309B-parameter sparse MoE (15B active per token) built around a deliberately aggressive hybrid attention schedule. Eight hybrid blocks each stack five sliding-window-attention layers (128-token window, 64 query / 8 KV heads) with one global-attention layer (64 query / 4 KV heads), giving 39 SWA and 9 GA layers; only the very first block is global with a dense FFN. A learnable attention sink bias in the softmax denominator — following OpenAI's gpt-oss — lets each head dump surplus probability mass, which the authors show is what lets the 5:1 ratio and 128-token window match or beat an all-global baseline on general, long-context, and reasoning benchmarks. Position comes from partial RoPE (first 64 QK dimensions); the GA RoPE base is raised from 640,000 to 5,000,000 when the context is extended from 32K to 256K, while SWA keeps base 10,000. Every MoE layer routes each token through 8 of 256 experts (no shared experts, per-expert width 2048) under a sequence auxiliary loss plus expert-bias updates. Training runs FP8 mixed precision over 27T tokens in three stages (general at 32K, code/reasoning-heavy mid-training, 256K context extension). The headline post-training contribution is MOPD, multi-teacher on-policy distillation: domain-specialized RL teachers supply dense token-level reverse-KL rewards combined with GRPO-style outcome rewards so one student absorbs every teacher's peak capability. The lightweight 0.33B MTP blocks (dense FFN + SWA) double as a self-speculative draft model — ~3.6-token acceptance length, 2.6x decoding speedup — and are open-sourced alongside the weights.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Learnable attention sink bias in the softmax denominator", "provenance": "stated", "ref": "§2.2 ('our implementation follows the design used in gpt-oss [agarwal2025gpt]')" }, "origin": { "value": "OpenAI gpt-oss", "provenance": "stated", "ref": "§2.2" } }, { "mechanism": { "value": "FP8 mixed-precision framework (BF16 attention output projections and embeddings/output head, FP32 router)", "provenance": "stated", "ref": "§2.1 ('similar to DeepSeek-V3 [liu2024deepseek]')" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) training objective", "provenance": "stated", "ref": "§2.3 (cites gloeckle2024better, liu2024deepseek, xia2025mimo)" }, "origin": { "value": "DeepSeek-V3 / MiMo-7B", "provenance": "stated", "ref": "§2.3" } }, { "mechanism": { "value": "GRPO-style outcome-reward advantages combined with distillation rewards", "provenance": "stated", "ref": "§4.4 (cites shao2024deepseekmath)" }, "origin": { "value": "DeepSeek (GRPO / DeepSeekMath)", "provenance": "stated", "ref": "§4.4" } } ], "influence_out": [], "notable_omissions": { "value": [ "Normalization type and pre/post placement never named (no RMSNorm/LayerNorm mention)", "FFN activation function never named (no SwiGLU/GELU)", "Vocabulary size and tokenizer not disclosed", "MoE router function not described (only bias/aux-loss hyperparameters)", "Training hardware and compute (FLOPs/GPU-hours) not disclosed", "Training parallelism strategy not disclosed", "No license named for the weights release", "Embedding/output-head tying not disclosed", "MoE expert granularity not discussed", "SWA/GA per-head dims (192/128) do not match hidden 4096 / 64 heads = 64, and the discrepancy is unexplained", "RL reward-model training details (beyond GRPO mention) deferred", "Context extension relies on RoPE base change only; no comparison to YaRN/NTK given" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "xiaomi-mimo-v2-flash-309b.json" }, { "slug": { "value": "zai-glm-4-5-air", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2508.06471", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/zai-glm-4-5-air/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "GLM-4.5 (ARC series)", "provenance": "stated", "ref": "Title; §1 ('This paper introduces two new models: GLM-4.5 and GLM-4.5-Air')" }, "variants": [ { "value": "GLM-4.5-Air (106B MoE, 12B active)", "provenance": "stated", "ref": "Abstract; Table 1" }, { "value": "GLM-4.5 (355B MoE, 32B active)", "provenance": "stated", "ref": "Abstract; Table 1" } ], "org": { "value": "Zhipu AI & Tsinghua University (GLM-4.5 Team)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2025-08-08", "provenance": "inferred", "note": "Report text is undated (only 'evaluated as of July 28, 2025' appears in Fig. 1); arXiv v1 submission date 2025-08-08 per export.arxiv.org API metadata (arXiv:2508.06471)" }, "report_url": { "value": "https://arxiv.org/abs/2508.06471", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "report never names a license for the weights release" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('We release both GLM-4.5 (355B parameters) and a compact version, GLM-4.5-Air (106B parameters)'); §1 (huggingface.co/zai-org/GLM-4.5); §5 ('We release the model weights of GLM-4.5 and GLM-4.5-Air')" } }, "scale": { "total_params": { "value": 106, "provenance": "stated", "ref": "Abstract; Table 1", "note": "Table 1 counts MTP-layer parameters but not word embeddings and the output layer" }, "active_params_per_token": { "value": 12, "provenance": "stated", "ref": "Abstract; Table 1" }, "layers": { "value": 46, "provenance": "derived", "note": "1 dense layer + 45 MoE layers (Table 1) = 46 backbone layers; the 1 MTP layer is a separate auxiliary layer" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 1 (Hidden Dim)" }, "ffn_inner_dim": { "value": 1408, "provenance": "stated", "ref": "Table 1 (MoE Intermediate Dim)", "note": "per-expert intermediate dim; the single dense layer uses Dense Intermediate Dim 10944 instead" }, "ffn_ratio": { "value": 0.34, "provenance": "derived", "note": "1408 / 4096 = 0.34375 (per-expert MoE ratio; the dense layer has ratio 10944/4096 = 2.67)" }, "attention_heads_q": { "value": 96, "provenance": "stated", "ref": "Table 1 (# Attention Heads)" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1 (# Key-Value Heads) — GQA" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 1 (Attention Head Dim)" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "vocabulary size never appears in the report" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed (parameter counts explicitly exclude embeddings and the output layer, Table 1, but tying is not stated)" }, "variants": [ { "name": { "value": "GLM-4.5", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 355, "provenance": "stated", "ref": "Table 1" }, "layers": { "value": 92, "provenance": "derived", "note": "3 dense + 89 MoE backbone layers (Table 1); MTP layer separate" }, "hidden_dim": { "value": 5120, "provenance": "stated", "ref": "Table 1" }, "ffn_inner_dim": { "value": 1536, "provenance": "stated", "ref": "Table 1 (MoE Intermediate Dim; dense layers use 12288)" }, "attention_heads_q": { "value": 96, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§2.4 (max sequence length extended to 131,072; 128K context limit in §4.2.3)" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "§1 ('GLM-4.5-Air is a smaller MoE model with 106B parameters'); §5 ('Both models adopt the MoE architecture'); Table 1", "note": "45/46 = 97.8% of backbone layers are MoE (1 dense layer, Table 1); MTP layer also MoE (§2.1)" }, "moe": { "expert_count": { "value": 128, "provenance": "stated", "ref": "Table 1 (# Experts (total) 128)", "note": "table mirrors the DeepSeek-V3 column convention where 'total' excludes the shared expert (1 shared expert listed separately)" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "Table 1 (# Experts Active Per Token)" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "Table 1 (# Shared Experts)" }, "routing": { "value": "sigmoid gates with loss-free balance routing", "provenance": "stated", "ref": "§2.1 ('In the GLM-4.5 series, we adopt the MoE architecture... We employ loss-free balance routing [40] and sigmoid gates for MoE layers [23]')" }, "load_balancing": { "value": "loss-free balance routing: expert bias update rate 0.001 for the first 15T tokens, then 0.0; auxiliary sequence-level balance loss with weight 0.0001", "provenance": "stated", "ref": "§2.1, §2.4 (series-level recipe)" }, "expert_granularity": { "value": "compact per-expert FFN (MoE intermediate dim 1408, 0.34× hidden) vs dense-layer intermediate 10944; report uses no fine-grained/grouped terminology", "provenance": "inferred", "note": "evidence: Table 1 (MoE Intermediate Dim 1408 vs Dense Intermediate Dim 10944) and §2.1 (width reduced relative to DeepSeek-V3/Kimi K2)" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2.1 ('Grouped-Query Attention with partial RoPE'); Table 1 (96 Q heads / 8 KV heads)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "stated", "ref": "§2.1 (GQA in the self-attention component; no per-layer hybrid described)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: report describes no learned attention over prior layer outputs; standard residual stack implied" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1 (partial RoPE); §2.4 (RoPE base frequency adjustment)" }, "rope_base": { "value": 1000000, "provenance": "stated", "ref": "§2.4 ('we also adjusted RoPE's base frequency from 10,000 to 1,000,000' when extending to 32K)", "note": "initial base was 10,000; raised to 1,000,000 at the 32K extension stage" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§2.1 (Grouped-Query Attention with partial RoPE)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "no YaRN/NTK/PI: context is grown by training at longer sequence lengths (4K → 32K → 128K) with the RoPE base frequency raised from 10,000 to 1,000,000 at the 32K stage", "ref": "§2.3, §2.4" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.4 (base-frequency change, not an interpolation/rescaling factor)" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§2.4 (max sequence length extended to 32,768 and 131,072 during mid-training)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.4 ('weights for RMSNorm' excluded from the Muon optimizer; series-level recipe)" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": false, "provenance": "stated", "ref": "Table 1 (QK-Norm: No)", "note": "QK-Norm is applied to GLM-4.5 but explicitly not to GLM-4.5-Air — the only stated architectural difference between the two" } }, "activation": { "value": null, "provenance": "unknown", "note": "FFN activation function never named anywhere in the report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described" }, "other": [] } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§2.4 (series-level recipe: max sequence length 4,096 during pre-training, extended to 32,768 and 131,072 in mid-training); §3.1 (overall SFT at max 128K)" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§4.2.3 (evaluation 'prevent exceeding the 128K context limit')" }, "extension_method": { "value": "trained at extended lengths: 4K → 32K → 128K across pre-/mid-training, with RoPE base frequency raised from 10,000 to 1,000,000 at the 32K stage", "provenance": "stated", "ref": "§2.3, §2.4" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer never described; vocabulary size not disclosed either" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": null, "provenance": "unknown", "note": "the abstract's 23T figure is stated for GLM-4.5's multi-stage training; no separate token budget is disclosed for GLM-4.5-Air" }, "data_composition": { "value": "series-level corpus (report describes the GLM-4.5 series jointly): webpages, social media, books, papers, and code repositories; web documents bucketed by quality (top bucket >3.2 epochs, lowest discarded, MinHash + SemDedup); multilingual from crawled web and Fineweb-2 with quality-classifier up-sampling; code from GitHub with tiered quality models and Fill-In-the-Middle; math & science up-sampled above an LLM-scored threshold; two-stage pre-training (general documents, then up-sampled code/math/science)", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "two-stage pre-training; mid-training stages: repo-level code (4K→32K), synthetic reasoning data, long-context & agent training (32K→128K); best-fit packing in mid-training only (series-level recipe)", "provenance": "stated", "ref": "§2.2, §2.3" }, "optimizer": { "value": "Muon for all parameters except word embeddings, biases, and RMSNorm weights (Newton-Schulz iterations N=5, momentum 0.95, scaled update RMS 0.2); weight decay 0.1; no dropout (series-level recipe)", "provenance": "stated", "ref": "§2.4" }, "lr_schedule": { "value": "cosine decay (chosen over warmup-stable-decay); warmup 0 → 2.5e-4, decaying to 2.5e-5 until the end of mid-training (series-level recipe)", "provenance": "stated", "ref": "§2.4" }, "batch_schedule": { "value": "batch size warmup: 16M → 64M tokens over the first 500B tokens, then constant (series-level recipe)", "provenance": "stated", "ref": "§2.4" }, "precision": { "value": "BF16 training with FP8 inference for RL rollouts (online block-wise FP8 quantization before rollout dispatch)", "provenance": "stated", "ref": "§3.5" }, "parallelism": { "value": "Megatron-based training (per Slime infra description); detailed TP/PP/EP/CP strategy not described", "provenance": "stated", "ref": "§3.5 (Fig. 10)", "note": "only the framework name is given; no parallelism configuration disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (cold-start SFT + overall SFT at 128K max context; series-level pipeline applying to both models)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§3.2 ('Our overall RL algorithm builds upon the GRPO [31] framework, excluding the KL loss term'); §3.4", "note": "agentic RL uses group-wise policy optimization; general RL combines rule-based, human (RLHF), and model-based (RLAIF) feedback (series-level pipeline)" }, "reasoning_training": { "value": "hybrid reasoning modes (thinking + non-thinking); reasoning RL with two-stage difficulty-based curriculum, single-stage RL at 64K max output length, dynamic sampling temperature, token-weighted mean loss for code RL; agentic RL (web-search + SWE) with process format penalty and iterative self-distillation; general RL (holistic, instruction-following, function-calling, pathology); test-time compute via interaction turns (series-level pipeline, §3.2–3.4)", "provenance": "stated", "ref": "§1, §3.2, §3.3, §3.4" }, "distillation": { "value": "self-distillation (no external teacher): Stage 1 domain experts (Reasoning, Agent, General chat) consolidated into a unified hybrid-reasoning model in Stage 2 via SFT distillation; iterative self-distillation in agentic RL (series-level pipeline)", "provenance": "stated", "ref": "§3, §3.3.2" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: all tasks, data, and benchmarks described in the report are textual; no vision/audio modality appears anywhere" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1–§4 (text-only model)" } }, "efficiency": { "kv_cache": { "value": "GQA with 8 KV heads shared across 96 query heads: KV cache proportional to 8 heads (12× reduction vs 96-head MHA)", "provenance": "derived", "note": "derived from attention_variant: 96 Q heads / 8 KV heads (Table 1); report itself does not discuss KV-cache size" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "§3.5", "note": "no shipped deployment quantization disclosed; FP8 appears only as online block-wise quantization for RL rollout inference" }, "speculative_dedup": { "value": "MTP (Multi-Token Prediction) MoE layer supporting speculative decoding during inference; MTP loss weight λ = 0.3 for the first 15T tokens, 0.1 after (series-level recipe)", "provenance": "stated", "ref": "§2.1, §2.4" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "§3.5", "note": "SGLang appears only as the RL rollout engine; no serving optimizations disclosed" } }, "evaluation": { "benchmarks": [ { "name": { "value": "TAU-Bench (Retail / Airline)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "77.9 / 60.8", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BFCL V3", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 76.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 21.3, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME 24 (Avg@32)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 89.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA (Avg@8)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 75.0, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 81.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MATH 500", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 98.1, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SciCode", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 37.3, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HLE", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 10.6, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LiveCodeBench (2407-2501)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 70.7, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SWE-bench Verified", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 57.6, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Terminal-Bench", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 30.0, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 87.4, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SimpleQA", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 14.5, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "IFEval (Prompt Strict)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 86.3, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SysBench (ISR)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 77.4, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MultiChallenge", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 42.5, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SafetyBench (average)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 87.8, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Novel logical reasoning (expert score)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 53.4, "provenance": "stated", "ref": "Table 11" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } } ] }, "contributions": { "claimed": [ { "text": { "value": "GLM-4.5-Air: a compact 106B-parameter MoE representing a significant leap among 100B-scale models, matching or exceeding Qwen3-235B-A22B and MiniMax-M1, and ranked 6th overall on the ARC benchmark set", "provenance": "stated", "ref": "§1, Fig. 1" }, "ref": { "value": "§1, Fig. 1", "provenance": "stated", "ref": "§1, Fig. 1" } }, { "text": { "value": "Both models lie on the Pareto frontier of SWE-bench Verified vs model parameters among open-source models", "provenance": "stated", "ref": "§1, Fig. 2" }, "ref": { "value": "§1, Fig. 2", "provenance": "stated", "ref": "§1, Fig. 2" } }, { "text": { "value": "Series-level: depth-over-width MoE design, 2.5× attention heads with QK-Norm (flagship only), loss-free balance routing with sigmoid gates, MoE MTP layer for speculative decoding, expert-model-iteration post-training with hybrid reasoning modes, and the Slime RL infrastructure", "provenance": "stated", "ref": "§2, §3, §3.5" }, "ref": { "value": "§2, §3, §3.5", "provenance": "stated", "ref": "§2, §3, §3.5" } } ], "assessment": { "value": "GLM-4.5-Air is the same block as GLM-4.5 at smaller scale (4096 hidden, 46 layers, 128 experts) with exactly one stated architectural difference: QK-Norm is absent (Table 1). The record exists separately from zai-glm-4-5 because its config differs, not because its block type differs — the task premise that Air is 'dense' is contradicted by the report, which explicitly calls it an MoE ('GLM-4.5-Air is a smaller MoE model with 106B parameters', §1; 45/46 MoE layers, Table 1). Relative to the 100B-scale atlas class (qwen-qwen3-moe 235B-A22B), Air is narrower and deeper with 96/8 GQA heads and per-expert width 1408; its 12B active params are well below Qwen3's 22B, which the report leverages as a parameter-efficiency claim. The QK-Norm ablation between flagship and Air is disclosed but never analyzed, which weakens the causal story for Air's lower scores.", "provenance": "inferred", "note": "comparison against atlas entries (zai-glm-4-5, qwen-qwen3-moe, deepseek-v3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "45/46 = 97.8% of backbone layers are MoE (1 dense layer, Table 1); MTP layer also MoE" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "GQA in every layer; uniform pattern, no hybrid (Table 1, §2.1)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (weights of both models released, §1/§5); pre-training data composition described (§2.2) but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "106B ≥ 100B (mechanical rule; the report itself frames Air as a '100B-scale' model)" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "same framing as the flagship: a compact generalist unifying agentic, reasoning, and coding abilities with hybrid thinking/non-thinking modes (§1)" } }, "prose": { "design_overview": { "value": "GLM-4.5-Air is the compact sibling in Zhipu AI's GLM-4.5 series: a 46-layer sparse MoE (45 MoE + 1 dense layer) with 106B total and 12B activated parameters, hidden dim 4096, 128 routed experts (8 active) plus one shared expert, and per-expert intermediate dim 1408 against 10944 in its single dense layer. It shares the flagship's block design — GQA with 96 query heads over 8 KV heads (head dim 128), partial RoPE with the base raised from 10,000 to 1,000,000 at the 32K context stage, sigmoid gating with loss-free bias-based load balancing plus a small sequence-level balance loss, RMSNorm, and a dedicated MoE MTP layer used for speculative decoding — with one notable exception: Table 1 marks QK-Norm as 'No' for Air, the only stated architectural difference from GLM-4.5. Training follows the same series recipe (two-stage pre-training, repo-level code / synthetic reasoning / long-context-agent mid-training, 4K→128K context growth, Muon optimizer, cosine decay, BF16 with FP8 rollouts), though the report gives no token budget specific to Air. Post-training is the shared expert-model-iteration pipeline: domain experts for Reasoning, Agent, and General chat are trained and then distilled via SFT into one hybrid-reasoning model with thinking and non-thinking modes. The report positions Air as a 100B-scale parameter-efficiency play, matching or exceeding Qwen3-235B-A22B and MiniMax-M1 on ARC benchmarks while activating only 12B parameters.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Sigmoid gating and loss-free (auxiliary-loss-free) balance routing for MoE layers", "provenance": "stated", "ref": "§2.1 (cites DeepSeek-V3 [23] and aux-loss-free balancing [40])" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Shared + routed expert MoE organization and MTP layer (Table 1 columns mirror DeepSeek-V3's)", "provenance": "stated", "ref": "§2.1, Table 1" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) layer", "provenance": "stated", "ref": "§2.1 (cites Gloeckle et al. [12])" }, "origin": { "value": "Multi-token prediction (Gloeckle et al. 2024)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "GRPO as the RL backbone, KL term excluded", "provenance": "stated", "ref": "§3.2 (cites DeepSeekMath [31])" }, "origin": { "value": "DeepSeekMath (DeepSeek)", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "Muon optimizer for hidden-layer parameters", "provenance": "stated", "ref": "§2.4 (cites Jordan et al. [21], Liu et al. [24])" }, "origin": { "value": "Muon (Jordan et al. 2024 / Liu et al. 2025)", "provenance": "stated", "ref": "§2.4" } }, { "mechanism": { "value": "Shared architecture and post-training pipeline with the flagship GLM-4.5 (same series, smaller scale, no QK-Norm)", "provenance": "stated", "ref": "§1, §2.1, Table 1" }, "origin": { "value": "GLM-4.5 (same report)", "provenance": "stated", "ref": "§1, Table 1" } } ], "influence_out": [ { "model_slug": { "value": "prime-intellect-intellect-3", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Base model and full architecture: SFT and RL run on top of the GLM-4.5-Air base (all block-level design inherited)", "provenance": "inferred", "note": "mechanism as stated in prime-intellect-intellect-3 report's lineage_in" } } ], "notable_omissions": { "value": [ "Air's own pre-training token budget not disclosed (23T stated only for GLM-4.5)", "The QK-Norm difference between flagship and Air (Table 1) is never analyzed or ablated in the text", "FFN activation function never named", "Tokenizer and vocabulary size not disclosed", "Input/output embedding tying not disclosed", "Normalization placement (pre/post) not described", "Training hardware and total compute (FLOPs/GPU-hours) not disclosed", "Parallelism strategy not described (only 'Megatron' named in Fig. 10)", "No license named for the weights release", "No human-evaluation, CC-Bench, or translation results for Air (those sections cover GLM-4.5 only)", "No long-context benchmark results despite the 128K window claim" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "zai-glm-4-5-air.json" }, { "slug": { "value": "zai-glm-4-5", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2508.06471", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/zai-glm-4-5/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "GLM-4.5 (ARC series)", "provenance": "stated", "ref": "Title; §5 ('the GLM-4.5 model series, including GLM-4.5 and GLM-4.5-Air')" }, "variants": [ { "value": "GLM-4.5 (355B MoE, 32B active)", "provenance": "stated", "ref": "Abstract; Table 1" }, { "value": "GLM-4.5-Air (106B MoE, 12B active)", "provenance": "stated", "ref": "Abstract; Table 1" } ], "org": { "value": "Zhipu AI & Tsinghua University (GLM-4.5 Team)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2025-08-08", "provenance": "inferred", "note": "Report text is undated (only 'evaluated as of July 28, 2025' appears in Fig. 1); arXiv v1 submission date 2025-08-08 per export.arxiv.org API metadata (arXiv:2508.06471)" }, "report_url": { "value": "https://arxiv.org/abs/2508.06471", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "report never names a license for the weights release" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('We release both GLM-4.5 (355B parameters) and a compact version, GLM-4.5-Air (106B parameters)'); §1 (huggingface.co/zai-org/GLM-4.5); §5 ('We release the model weights of GLM-4.5 and GLM-4.5-Air')" } }, "scale": { "total_params": { "value": 355, "provenance": "stated", "ref": "Abstract; Table 1", "note": "Table 1 counts MTP-layer parameters but not word embeddings and the output layer" }, "active_params_per_token": { "value": 32, "provenance": "stated", "ref": "Abstract; Table 1" }, "layers": { "value": 92, "provenance": "derived", "note": "3 dense layers + 89 MoE layers (Table 1) = 92 backbone layers; the 1 MTP layer is a separate auxiliary layer (like DeepSeek-V3's MTP, not part of the main stack)" }, "hidden_dim": { "value": 5120, "provenance": "stated", "ref": "Table 1 (Hidden Dim)" }, "ffn_inner_dim": { "value": 1536, "provenance": "stated", "ref": "Table 1 (MoE Intermediate Dim)", "note": "per-expert intermediate dim; the 3 dense layers use Dense Intermediate Dim 12288 instead" }, "ffn_ratio": { "value": 0.3, "provenance": "derived", "note": "1536 / 5120 = 0.30 (per-expert MoE ratio; dense layers have ratio 12288/5120 = 2.4)" }, "attention_heads_q": { "value": 96, "provenance": "stated", "ref": "Table 1 (# Attention Heads); §2.1 (96 heads for a 5120 hidden dimension)" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1 (# Key-Value Heads) — GQA" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table 1 (Attention Head Dim)" }, "vocab_size": { "value": null, "provenance": "unknown", "note": "vocabulary size never appears in the report" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed (parameter counts explicitly exclude embeddings and the output layer, Table 1, but tying is not stated)" }, "variants": [ { "name": { "value": "GLM-4.5-Air", "provenance": "stated", "ref": "Table 1" }, "total_params": { "value": 106, "provenance": "stated", "ref": "Table 1" }, "layers": { "value": 46, "provenance": "derived", "note": "1 dense + 45 MoE backbone layers (Table 1); MTP layer separate" }, "hidden_dim": { "value": 4096, "provenance": "stated", "ref": "Table 1" }, "ffn_inner_dim": { "value": 1408, "provenance": "stated", "ref": "Table 1 (MoE Intermediate Dim; dense layer uses 10944)" }, "attention_heads_q": { "value": 96, "provenance": "stated", "ref": "Table 1" }, "attention_heads_kv": { "value": 8, "provenance": "stated", "ref": "Table 1" }, "context_length": { "value": 131072, "provenance": "stated", "ref": "§2.4 (max sequence length extended to 131,072; 128K context limit in §4.2.3)" } } ] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "stated", "ref": "§2.1 ('we adopt the MoE architecture'); Table 1", "note": "89/92 = 96.7% of backbone layers are MoE (3 dense layers, Table 1); MTP layer is also an MoE layer (§2.1)" }, "moe": { "expert_count": { "value": 160, "provenance": "stated", "ref": "Table 1 (# Experts (total) 160)", "note": "table mirrors the DeepSeek-V3 column convention where 'total' excludes the shared expert (1 shared expert listed separately)" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "Table 1 (# Experts Active Per Token)" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "Table 1 (# Shared Experts)" }, "routing": { "value": "sigmoid gates with loss-free balance routing", "provenance": "stated", "ref": "§2.1 ('We employ loss-free balance routing [40] and sigmoid gates for MoE layers [23]')" }, "load_balancing": { "value": "loss-free balance routing: expert bias update rate 0.001 for the first 15T tokens, then 0.0; auxiliary sequence-level balance loss with weight 0.0001 to avoid extreme imbalance within a single sequence", "provenance": "stated", "ref": "§2.1, §2.4" }, "expert_granularity": { "value": "compact per-expert FFN (MoE intermediate dim 1536, 0.3× hidden) vs dense-layer intermediate 12288; report uses no fine-grained/grouped terminology", "provenance": "inferred", "note": "evidence: Table 1 (MoE Intermediate Dim 1536 vs Dense Intermediate Dim 12288) and §2.1 (width reduced relative to DeepSeek-V3/Kimi K2)" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "§2.1 ('Grouped-Query Attention with partial RoPE'); Table 1 (96 Q heads / 8 KV heads)" }, "attention_layer_pattern": { "value": "uniform", "provenance": "stated", "ref": "§2.1 (GQA in the self-attention component; no per-layer hybrid described)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "absence rule: report describes no learned attention over prior layer outputs; standard residual stack implied" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§2.1 (partial RoPE); §2.4 (RoPE base frequency adjustment)" }, "rope_base": { "value": 1000000, "provenance": "stated", "ref": "§2.4 ('we also adjusted RoPE's base frequency from 10,000 to 1,000,000' when extending to 32K)", "note": "initial base was 10,000; raised to 1,000,000 at the 32K extension stage" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "§2.1 (Grouped-Query Attention with partial RoPE)" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "no YaRN/NTK/PI: context is grown by training at longer sequence lengths (4K → 32K → 128K) with the RoPE base frequency raised from 10,000 to 1,000,000 at the 32K stage", "ref": "§2.3, §2.4" }, "factor": { "value": null, "provenance": "n/a", "ref": "§2.4 (base-frequency change, not an interpolation/rescaling factor)" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "§2.4 (max sequence length extended to 32,768 and 131,072 during mid-training)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§2.4 ('weights for RMSNorm' excluded from the Muon optimizer)" }, "placement": { "value": null, "provenance": "unknown", "note": "pre/post norm placement not described" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "§2.1 ('We also incorporate QK-Norm [15] to stabilize the range of attention logits'); Table 1 (QK-Norm: Yes)" } }, "activation": { "value": null, "provenance": "unknown", "note": "FFN activation function never named anywhere in the report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "absence rule: no logit softcapping described" }, "other": [] } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "§2.4 (max sequence length 4,096 during pre-training, extended to 32,768 and 131,072 in mid-training); §3.1 (overall SFT at max 128K)" }, "deployed_length": { "value": 131072, "provenance": "stated", "ref": "§4.2.3 (evaluation 'prevent exceeding the 128K context limit')" }, "extension_method": { "value": "trained at extended lengths: 4K → 32K → 128K across pre-/mid-training, with RoPE base frequency raised from 10,000 to 1,000,000 at the 32K stage", "provenance": "stated", "ref": "§2.3, §2.4" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer never described; vocabulary size not disclosed either" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 23, "provenance": "stated", "ref": "Abstract ('multi-stage training on 23T tokens')" }, "data_composition": { "value": "corpus of webpages, social media, books, papers, and code repositories: web documents bucketed by quality scores (Nemotron-CC-inspired; top bucket contributes over 3.2 epochs, lowest bucket discarded; MinHash + SemDedup deduplication); multilingual documents from crawled web and Fineweb-2 with educational-utility classifier up-sampling; code from GitHub and code hosting platforms with tiered quality models (high/medium/low) and Fill-In-the-Middle objective on all source code; math & science documents scored by an LLM and up-sampled above a threshold; two-stage pre-training: stage 1 general web documents, stage 2 up-sampled code, math, and science", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "two-stage pre-training (general documents, then up-sampled code/math/science); mid-training stages: repo-level code training (4K→32K, concatenated same-repo files, filtered issues/PRs/commits in diff format), synthetic reasoning data training, long-context & agent training (32K→128K, up-sampled long documents, synthetic agent trajectories); best-fit packing used only in mid-training (random truncation kept in pre-training)", "provenance": "stated", "ref": "§2.2, §2.3" }, "optimizer": { "value": "Muon for all parameters except word embeddings, biases, and RMSNorm weights (Newton-Schulz iterations N=5, momentum 0.95, scaled Muon update RMS 0.2); weight decay 0.1; no dropout", "provenance": "stated", "ref": "§2.4" }, "lr_schedule": { "value": "cosine decay (chosen over warmup-stable-decay after experiments); warmup 0 → 2.5e-4, decaying to 2.5e-5 until the end of mid-training", "provenance": "stated", "ref": "§2.4" }, "batch_schedule": { "value": "batch size warmup: gradually increased from 16M to 64M tokens over the first 500B tokens, then constant", "provenance": "stated", "ref": "§2.4" }, "precision": { "value": "BF16 training with FP8 inference for RL rollouts (online block-wise FP8 quantization of parameters before rollout dispatch)", "provenance": "stated", "ref": "§3.5" }, "parallelism": { "value": "Megatron-based training (per Slime infra description); detailed TP/PP/EP/CP strategy not described", "provenance": "stated", "ref": "§3.5 (Fig. 10: 'Training (Megatron) – handles the main training process')", "note": "only the framework name is given; no parallelism configuration disclosed" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (cold-start SFT with extended CoT responses; overall SFT on millions of samples covering reasoning, general chat, agentic, and long-context tasks at 128K max context; function-call template with XML-like special token tags; rejection sampling; prompt selection with response-level scaling)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§3.2 ('Our overall RL algorithm builds upon the GRPO [31] framework, excluding the KL loss term'); §3.4 ('crucial during GRPO training')", "note": "agentic RL uses a group-wise policy optimization objective (K traces, mean-reward baseline, environment feedback excluded from loss); general RL combines rule-based feedback, human feedback (RLHF reward model on preference annotations), and model-based feedback (RLAIF rubrics)" }, "reasoning_training": { "value": "hybrid reasoning modes (thinking mode for complex reasoning/agentic tasks, non-thinking mode for instant responses); reasoning RL: two-stage difficulty-based curriculum (stage 2 problems verified pass@8==0, pass@512>>0), single-stage RL directly at 64K max output length (multi-stage progressive-length RL found inferior), dynamic sampling temperature (raised on convergence, capped by ≤1% validation drop), token-weighted mean loss for code RL, expert-verified MCQ-only data for science RL; agentic RL on web-search and SWE tasks with process format penalty and iterative self-distillation; test-time compute scaled via interaction turns; general RL: holistic RL (~5,000 prompts, 7/33/139 category taxonomy), instruction-following RL (7 major/151 minor constraint taxonomy, rule+RM+critique feedback), function-calling RL (step-wise rule-based + end-to-end multi-turn), pathology RL", "provenance": "stated", "ref": "§1, §3.2, §3.3, §3.4" }, "distillation": { "value": "self-distillation (no external teacher): Stage 1 trains three domain experts (Reasoning, Agent, General chat); Stage 2 overall SFT distills expert outputs into one unified hybrid-reasoning generalist; agentic RL additionally uses iterative self-distillation (cold-start SFT data replaced by RL-trained responses before further RL)", "provenance": "stated", "ref": "§3, §3.3.2" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "evidence: all tasks, data, and benchmarks described in the report are textual (ARC, chat, translation, safety); no vision/audio modality appears anywhere. (A separate GLM-4.5V vision variant exists outside this report.)" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1–§4 (text-only model)" } }, "efficiency": { "kv_cache": { "value": "GQA with 8 KV heads shared across 96 query heads: KV cache proportional to 8 heads (12× reduction vs 96-head MHA)", "provenance": "derived", "note": "derived from attention_variant: 96 Q heads / 8 KV heads (Table 1); report itself does not discuss KV-cache size" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "§3.5", "note": "no shipped deployment quantization disclosed; FP8 appears only as online block-wise quantization for RL rollout inference" }, "speculative_dedup": { "value": "MTP (Multi-Token Prediction) MoE layer supporting speculative decoding during inference; MTP loss weight λ = 0.3 for the first 15T tokens, 0.1 after", "provenance": "stated", "ref": "§2.1, §2.4" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "§3.5", "note": "SGLang appears only as the RL rollout engine; no serving optimizations disclosed" } }, "evaluation": { "benchmarks": [ { "name": { "value": "TAU-Bench (Retail / Airline)", "provenance": "stated", "ref": "Table 3" }, "value": { "value": "79.7 / 60.4", "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BFCL V3", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 77.8, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "BrowseComp", "provenance": "stated", "ref": "Table 3" }, "value": { "value": 26.4, "provenance": "stated", "ref": "Table 3" }, "ref": { "value": "Table 3", "provenance": "stated", "ref": "Table 3" } }, { "name": { "value": "AIME 24 (Avg@32)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 91.0, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "GPQA (Avg@8)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 79.1, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 84.6, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "MATH 500", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 98.2, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SciCode", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 41.7, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "HLE", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 14.4, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "LiveCodeBench (2407-2501)", "provenance": "stated", "ref": "Table 4" }, "value": { "value": 72.9, "provenance": "stated", "ref": "Table 4" }, "ref": { "value": "Table 4", "provenance": "stated", "ref": "Table 4" } }, { "name": { "value": "SWE-bench Verified", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 64.2, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "Terminal-Bench", "provenance": "stated", "ref": "Table 5" }, "value": { "value": 37.5, "provenance": "stated", "ref": "Table 5" }, "ref": { "value": "Table 5", "provenance": "stated", "ref": "Table 5" } }, { "name": { "value": "MMLU", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 90.0, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SimpleQA", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 26.4, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "IFEval (Prompt Strict)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 86.1, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SysBench (ISR)", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 81.0, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "MultiChallenge", "provenance": "stated", "ref": "Table 6" }, "value": { "value": 52.8, "provenance": "stated", "ref": "Table 6" }, "ref": { "value": "Table 6", "provenance": "stated", "ref": "Table 6" } }, { "name": { "value": "SafetyBench (average)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 89.9, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "CC-Bench tool-calling success rate", "provenance": "stated", "ref": "§4.3.2 (Fig. 13)" }, "value": { "value": 90.6, "provenance": "stated", "ref": "§4.3.2" }, "ref": { "value": "§4.3.2", "provenance": "stated", "ref": "§4.3.2" } }, { "name": { "value": "Novel logical reasoning (expert score)", "provenance": "stated", "ref": "Table 11" }, "value": { "value": 62.0, "provenance": "stated", "ref": "Table 11" }, "ref": { "value": "Table 11", "provenance": "stated", "ref": "Table 11" } }, { "name": { "value": "Challenging translation (human score 0-3)", "provenance": "stated", "ref": "Table 12" }, "value": { "value": 1.71, "provenance": "stated", "ref": "Table 12" }, "ref": { "value": "Table 12", "provenance": "stated", "ref": "Table 12" } } ] }, "contributions": { "claimed": [ { "text": { "value": "An open-source MoE foundation model unifying agentic, reasoning, and coding (ARC) capabilities, with a hybrid reasoning method supporting both thinking and direct-response modes; ranked 3rd overall and 2nd on agentic benchmarks among all evaluated models", "provenance": "stated", "ref": "Abstract, §1" }, "ref": { "value": "Abstract, §1", "provenance": "stated", "ref": "Abstract, §1" } }, { "text": { "value": "Depth-over-width MoE design: width (hidden dim and routed-expert count) reduced and height (layer count) increased relative to DeepSeek-V3 and Kimi K2, because deeper models exhibited better reasoning capacity", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "2.5× more attention heads (96 heads for a 5120 hidden dimension) plus QK-Norm: the higher head count does not improve training loss but consistently improves reasoning benchmarks such as MMLU and BBH", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Loss-free balance routing with sigmoid gates for MoE layers, with an auxiliary sequence-level balance loss to prevent extreme intra-sequence imbalance", "provenance": "stated", "ref": "§2.1, §2.4" }, "ref": { "value": "§2.1, §2.4", "provenance": "stated", "ref": "§2.1, §2.4" } }, { "text": { "value": "An MoE layer as the MTP (Multi-Token Prediction) layer for both models, supporting speculative decoding during inference", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Expert model iteration post-training: three domain experts (Reasoning, Agent, General chat) consolidated via self-distillation into a unified hybrid-reasoning generalist", "provenance": "stated", "ref": "§3" }, "ref": { "value": "§3", "provenance": "stated", "ref": "§3" } }, { "text": { "value": "Novel function-call template encapsulating call keys and values in XML-like special token tags, substantially reducing character escaping for code segments without hurting function-call execution", "provenance": "stated", "ref": "§3.1" }, "ref": { "value": "§3.1", "provenance": "stated", "ref": "§3.1" } }, { "text": { "value": "Reasoning-RL recipes validated on a smaller model: two-stage difficulty-based curriculum, single-stage RL directly at 64K output length (multi-stage progressive-length RL found inferior), dynamic sampling temperature, and token-weighted mean loss for code RL", "provenance": "stated", "ref": "§3.2" }, "ref": { "value": "§3.2", "provenance": "stated", "ref": "§3.2" } }, { "text": { "value": "Slime, an open-source RL infrastructure supporting colocated synchronous and disaggregated asynchronous modes, BF16 training with FP8 rollout inference, and decoupled agentic rollout engines (Docker-based, unified HTTP endpoint, centralized data pool)", "provenance": "stated", "ref": "§3.5" }, "ref": { "value": "§3.5", "provenance": "stated", "ref": "§3.5" } }, { "text": { "value": "Open-sourced evaluation toolkit (glm-simple-evals) for reproducibility of benchmark results", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } } ], "assessment": { "value": "Architecturally a conservative DeepSeek-V3-style MoE (sigmoid gating, loss-free bias-based balancing, one shared expert, MTP layer) with three notable deltas: GQA with 8 KV heads plus QK-Norm instead of DeepSeek-V3's MLA (the only atlas MoE besides K3 with QK-normed global attention), a depth-over-width geometry (5120 hidden, 92 layers, 160 experts vs DeepSeek-V3's 7168/61/256), and the Muon optimizer. Nothing in the block is genuinely new relative to atlas peers — kimi-k3's KDA/AttnRes axes are absent here; the report's novelty is concentrated in post-training (domain-expert iteration with self-distillation into a hybrid thinking/non-thinking generalist) and in the unusually aggressive head-count/QK-norm choice. The 3rd-overall/2nd-agentic ranking claims are internal to the report. The activation function, tokenizer, vocab size, and training compute are all undisclosed, which limits comparability with peers.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, moonshot-kimi-k3, qwen-qwen3-moe, deepseek-r1) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "89/92 = 96.7% of backbone layers are MoE (3 dense layers, Table 1); MTP layer also MoE" }, "sequence_mixing": { "value": "full-attention", "provenance": "derived", "note": "GQA in every layer; uniform pattern, no hybrid (Table 1, §2.1)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true (HuggingFace/GitHub release); pre-training data composition described (§2.2) but not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "355B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a generalist unifying agentic, reasoning, and coding ('measure of a truly generalist model', §1); ranked 3rd overall among all evaluated models. Secondary intent: agentic/reasoning emphasis with hybrid thinking modes, noted in prose." } }, "prose": { "design_overview": { "value": "GLM-4.5 is Zhipu AI's first MoE model: a 92-layer stack (89 MoE + 3 dense layers) with 355B total and 32B activated parameters, deliberately made deeper and narrower than DeepSeek-V3 or Kimi K2 (5120 hidden dim, 160 routed experts with 8 active plus 1 shared expert, per-expert intermediate dim 1536 against 12288 in the dense layers) because the team found depth helped reasoning. Attention is GQA with an unusual 96 query heads and only 8 KV heads (head dim 128), with QK-Norm on the flagship to stabilize attention logits and partial RoPE whose base is raised from 10,000 to 1,000,000 at the 32K context stage; context grows 4K → 32K → 128K across pre- and mid-training. Routing uses sigmoid gates with DeepSeek-V3-style loss-free bias balancing (bias update rate 0.001 for the first 15T tokens, then 0) plus a small sequence-level balance loss; the MTP layer is itself an MoE layer and doubles as the speculative-decoding draft. Training runs 23T tokens with the Muon optimizer (N=5 Newton-Schulz steps, weight decay 0.1, no dropout) under cosine decay, batch-size warmup 16M→64M tokens, and BF16 with FP8 rollout inference. Post-training is a two-stage expert iteration: three domain experts (Reasoning, Agent, General chat) are trained with SFT + RL (GRPO-family reasoning RL at a single 64K output length, agentic RL with iterative self-distillation, holistic/instruction/function-calling/pathology general RL), then distilled via SFT into one unified model that can answer in thinking or non-thinking mode — the 'hybrid reasoning' that defines the ARC series.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Sigmoid gating for MoE layers", "provenance": "stated", "ref": "§2.1 (cites DeepSeek-V3 [23])" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Loss-free (auxiliary-loss-free) balance routing with expert biases", "provenance": "stated", "ref": "§2.1 (cites 'Auxiliary-loss-free load balancing strategy for mixture-of-experts' [40])" }, "origin": { "value": "DeepSeek (aux-loss-free balancing line)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Shared + routed expert MoE organization and MTP-layer parameter counting convention (Table 1 columns mirror DeepSeek-V3's)", "provenance": "stated", "ref": "§2.1, Table 1 (cites DeepSeek-V3 [23])" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "QK-Norm for attention-logit stabilization", "provenance": "stated", "ref": "§2.1 (cites Henry et al., 'Query-key normalization for transformers' [15])" }, "origin": { "value": "Query-Key Normalization for Transformers (Henry et al. 2020)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) layer", "provenance": "stated", "ref": "§2.1 (cites Gloeckle et al. [12])" }, "origin": { "value": "Multi-token prediction (Gloeckle et al. 2024)", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "GRPO (group relative policy optimization) as the RL backbone, KL term excluded", "provenance": "stated", "ref": "§3.2 (cites DeepSeekMath [31])" }, "origin": { "value": "DeepSeekMath (DeepSeek)", "provenance": "stated", "ref": "§3.2" } }, { "mechanism": { "value": "Muon optimizer for hidden-layer parameters", "provenance": "stated", "ref": "§2.4 (cites Jordan et al. [21], Liu et al. [24])" }, "origin": { "value": "Muon (Jordan et al. 2024 / Liu et al. 2025)", "provenance": "stated", "ref": "§2.4" } }, { "mechanism": { "value": "Quality-bucketed web corpus construction with up-sampling of high-quality buckets", "provenance": "stated", "ref": "§2.2 (cites Nemotron-CC [32], SemDedup [1], Fineweb-2 [27])" }, "origin": { "value": "Nemotron-CC / SemDedup / Fineweb-2", "provenance": "stated", "ref": "§2.2" } } ], "influence_out": [ { "model_slug": { "value": "zai-glm-4-5-air", "provenance": "inferred", "note": "authored by atlas analyst" }, "mechanism": { "value": "Shared architecture and post-training pipeline with the flagship GLM-4.5 (same series, smaller scale, no QK-Norm)", "provenance": "inferred", "note": "mechanism as stated in zai-glm-4-5-air report's lineage_in" } } ], "notable_omissions": { "value": [ "FFN activation function never named anywhere in the report", "Tokenizer and vocabulary size not disclosed", "Input/output embedding tying not disclosed", "Normalization placement (pre/post) not described", "Training hardware and total compute (FLOPs/GPU-hours) not disclosed", "Parallelism strategy (TP/PP/EP/CP) not described — only 'Megatron' is named in the infra diagram (Fig. 10)", "No license named for the weights release", "GLM-4.5-Air's own pre-training token budget not disclosed (23T is stated for GLM-4.5)", "SFT dataset size given only as 'millions of samples'; most RL data sizes/rollout counts undisclosed", "No long-context benchmark results (e.g. RULER-style) despite the 128K window claim", "Expert granularity terminology (fine-grained vs grouped) absent; per-expert width must be inferred from Table 1", "GLM-4.5-Air receives no human-evaluation or translation coverage (those sections cover GLM-4.5 only)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "zai-glm-4-5.json" }, { "slug": { "value": "zai-glm-5", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2602.15763", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/zai-glm-5/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "GLM-5", "provenance": "stated", "ref": "Title" }, "variants": [], "org": { "value": "Zhipu AI & Tsinghua University (GLM-5 Team)", "provenance": "stated", "ref": "Title page byline" }, "release_date": { "value": "2026-02-17", "provenance": "inferred", "note": "Report text is undated (GDPval-AA Elo 'recorded on 15th Feb., 2026' is an evaluation date, not a publication date); arXiv v1 submission date 2026-02-17 per export.arxiv.org API metadata (arXiv:2602.15763)" }, "report_url": { "value": "https://arxiv.org/abs/2602.15763", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "no license named in the report (only a github.com/zai-org/GLM-5 release link, §Abstract)" }, "open_weights": { "value": true, "provenance": "stated", "ref": "Abstract ('Code, models, and more information are available at https://github.com/zai-org/GLM-5'); §1 ('We release GLM-5 to the community'); §1 (first open weights model to score 50 on the AA Intelligence Index v4.0); §7 ('By open-sourcing the model')" } }, "scale": { "total_params": { "value": 744, "provenance": "stated", "ref": "§2.1 ('a 744B parameter model'), Table 10 (# Total Parameters 744B)" }, "active_params_per_token": { "value": 40, "provenance": "stated", "ref": "§2.1 ('40B active parameters'), Table 10 (# Activated Parameters 40B)" }, "layers": { "value": 78, "provenance": "derived", "note": "Table 10 sums to 78 backbone transformer layers (3 Dense Layers + 75 MoE Layers; 1 MTP layer counted separately). Prose §2.1 says 'reduces its layer count to 80' — the report does not reconcile the two counts." }, "hidden_dim": { "value": 6144, "provenance": "stated", "ref": "Table 10 (Hidden Dim 6144)" }, "ffn_inner_dim": { "value": 2048, "provenance": "stated", "ref": "Table 10 (MoE Intermediate Dim 2048)", "note": "per-expert MoE intermediate dim; the 3 dense layers use the separate Dense Intermediate Dim 12288" }, "ffn_ratio": { "value": 0.33, "provenance": "derived", "note": "per-expert MoE ratio: 2048 / 6144 = 0.333; dense layers: 12288 / 6144 = 2.0 (both from Table 10)" }, "attention_heads_q": { "value": 64, "provenance": "stated", "ref": "Table 10 (# Attention Heads 64)" }, "attention_heads_kv": { "value": null, "provenance": "n/a", "ref": "§2.1, Table 10 (# Key-Value Heads '-')", "note": "MLA layers compress KV into a low-rank latent (no KV heads); Table 10 leaves KV heads blank" }, "head_dim": { "value": 192, "provenance": "stated", "ref": "Table 10 (QK Head Dim 192)", "note": "QK head dim stated; V Head Dim is 256 (Table 10); MLA latent KV cache is 576-dim (KV LoRA 512 + 64, §2.1); heads are reduced by 1/3 vs DeepSeek-V3's MLA ('increase the head dimension from 192 to 256 and decrease the number of attention heads by 1/3', §2.1)" }, "vocab_size": { "value": 154880, "provenance": "stated", "ref": "Table 10 (Vocabulary Size 154880)" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "not disclosed; Table 10 note says parameter counts exclude word embeddings and the output layer" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "75 of 78 backbone layers (96.2%) are MoE layers per Table 10 (3 dense layers); §1 confirms 'standard MoE architecture' + DSA attention" }, "moe": { "expert_count": { "value": 255, "provenance": "derived", "note": "Table 10 '# Experts (total) 256' minus 1 shared expert = 255 routed experts (same reading as GLM-4.5's 160 = 159 routed + 1 shared)" }, "experts_per_token": { "value": 8, "provenance": "stated", "ref": "Table 10 (# Routed Experts 8)", "note": "interpreted as routed experts activated per token, consistent with GLM-4.5's identical row (160 total / 8 routed / 1 shared)" }, "shared_experts": { "value": 1, "provenance": "stated", "ref": "Table 10 (# Shared Experts 1)" }, "routing": { "value": null, "provenance": "unknown", "note": "router function (gating) not described in the report; only the expert count/organization appears (Table 10)" }, "load_balancing": { "value": null, "provenance": "unknown", "note": "no load-balancing mechanism disclosed" }, "expert_granularity": { "value": null, "provenance": "unknown", "note": "not described (no fine-grained/shared-expert-width detail beyond Table 10 counts)" } }, "attention_variant": { "value": "MLA", "provenance": "stated", "ref": "§2.1 (Multi-latent Attention), Table 10 (Q LoRA Dim 2048, KV LoRA Dim 512)", "note": "MLA-256 variant: 64 heads, QK head dim 192, V head dim 256, 576-dim latent KV cache (KV LoRA 512 + 64); attention is computed sparsely over DSA-retrieved tokens on all layers (see attention_layer_pattern)" }, "attention_layer_pattern": { "value": "uniform — DeepSeek Sparse Attention (DSA) applied to all layers: a lightning indexer (32 heads, head dim 128, top-k = 2048 KV entries per query) selects relevant keys/values and attention is computed sparsely over the retrieved subset; no dense-attention or sliding-window alternation", "provenance": "stated", "ref": "§2.1.1, §3.2 ('DSA is lossless by construction ... enabling application to all layers'), Table 10 (# Indexer Attn Heads 32, # Indexer Head Dim 128)" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: the report describes no attention over prior layer outputs; standard residual architecture implied throughout" }, "position_encoding": { "method": { "value": null, "provenance": "unknown", "note": "positional encoding never named in the report (no RoPE/NoPE/ALiBi mention); the 576-dim MLA latent (KV LoRA 512 + 64) is consistent with a 64-dim RoPE component, but the report does not state it" }, "rope_base": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "partial_rope": { "value": null, "provenance": "unknown", "note": "not disclosed" }, "extension": { "method": { "value": "progressive mid-training context curriculum (32K/1T tokens → 128K/500B → 200K/50B stages) followed by DSA warm-up (1,000 steps at 202,752-token sequences) and sparse adaptation (20B tokens); no positional-encoding modification disclosed", "provenance": "stated", "ref": "§2.3, §2.1.1" }, "factor": { "value": null, "provenance": "unknown", "note": "no rescaling/interpolation factor disclosed" }, "target_length": { "value": 200000, "provenance": "stated", "ref": "§2.3 (200K stage), §3.1 (SFT extends max context to 202,752 tokens)" } } }, "normalization": { "type": { "value": null, "provenance": "unknown", "note": "normalization type not described in the report" }, "placement": { "value": null, "provenance": "unknown", "note": "not described" }, "qk_norm": { "value": false, "provenance": "inferred", "note": "Absence rule: attention projections are described at the weight level (Muon Split on WUQ/WUK/WUV, QK/V head dims, §2.1) without any QK-normalisation mention" } }, "activation": { "value": null, "provenance": "unknown", "note": "activation function not stated in the report" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "Absence rule: no attention-logit softcapping described; §2.1 instead reports logit-scale stability from Muon Split 'without any clipping strategy'" }, "other": [ { "value": "Muon Split: per-head matrix orthogonalization of the MLA query/key/value up-projections, letting projection weights for different heads update at different scales; keeps attention-logit scale stable during pre-training without any clipping strategy", "provenance": "stated", "ref": "§2.1" }, { "value": "Deterministic torch.topk as the DSA indexer top-k operator (instead of non-deterministic CUDA/TileLang top-k) to eliminate training-inference mismatch during RL; indexer parameters frozen during RL", "provenance": "stated", "ref": "§3.2 (DSA RL insights)" }, { "value": "INT4 QAT with a quantization kernel used for both training and offline weight quantization that is bitwise-identical between training and inference", "provenance": "stated", "ref": "§2.4.3" } ] } }, "context": { "trained_length": { "value": 200000, "provenance": "stated", "ref": "§2.3 (mid-training extends the context window across 32K/128K/200K stages), §3.1 (SFT extends maximum context length to 202,752 tokens), §2.1.1 (DSA warm-up trains 202,752-token sequences)", "note": "pre-training starts at 4K; mid-training reaches 200K; SFT/eval operate at 202,752-token windows" }, "deployed_length": { "value": 200000, "provenance": "stated", "ref": "Appendix B.2 (SWE-bench evaluated 'with a 200K context window'), §6.1.1 (max generation 202,752 tokens for HLE-with-tools)" }, "extension_method": { "value": "progressive mid-training curriculum (32K → 128K → 200K) plus DSA sparse adaptation; no positional-encoding rescaling disclosed", "provenance": "stated", "ref": "§2.3, §2.1.1" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "tokenizer algorithm never described; only the 154,880 vocab size appears (Table 10)" }, "notes": { "value": null, "provenance": "unknown" } }, "training": { "tokens": { "value": 28.5, "provenance": "stated", "ref": "§2 ('totaling 28.5 trillion tokens for the base model'), §1 ('we extend the training token budget to 28.5T tokens')", "note": "§1 separately mentions the corpus 'began with a massive 27 trillion token corpus'; 28.5T is the total budget across all base-model stages" }, "data_composition": { "value": "Web: refined GLM-4.5 pipeline with an additional DCLM sentence-embedding classifier and a World Knowledge classifier (Wikipedia/LLM-labeled) for long-tail knowledge. Code: refreshed snapshots from code hosts plus code-containing web pages (+28% fuzzily deduplicated unique tokens), Software Heritage metadata fixes, and dedicated classifiers for low-resource programming languages (Scala, Swift, Lua, etc.). Math & science: LLM-scored documents from webpages/books/papers with chunk-and-aggregate scoring for long documents; synthetic/AI-generated/template data strictly filtered out", "provenance": "stated", "ref": "§2.2" }, "curriculum": { "value": "two-stage base training (pre-training prioritizing code and reasoning early on, then mid-training for agentic and long-context capacity) with progressive context extension 32K (1T tokens) → 128K (500B) → 200K (50B); long-context data (natural + synthetic interleaved packing inspired by NextLong/EntropyLong, MRCR-like data at 200K) upsampled at later stages; DSA warm-up (1,000 steps, 14 sequences of 202,752 tokens, lr 5e-3 → 2e-4) then sparse adaptation on 20B tokens (constant lr 1e-5)", "provenance": "stated", "ref": "§2, §2.3, §2.1.1" }, "optimizer": { "value": "Muon with Muon Split (per-head Newton-Schulz-style matrix orthogonalization applied to independent per-head up-projection matrices); zero-redundant distributed communication variant", "provenance": "stated", "ref": "§2.1, §2.4.1, Appendix A ('we follow the setting of GLM-4.5, including the Muon optimizer')" }, "lr_schedule": { "value": "pre-training: warmup from 0 to 2e-4 then decay to 4e-5 (cosine per GLM-4.5 setting); mid-training: linear decrease 4e-5 → 1e-5; DSA warm-up: 5e-3 → 2e-4; DSA sparse adaptation: constant 1e-5", "provenance": "stated", "ref": "Appendix A" }, "batch_schedule": { "value": null, "provenance": "unknown", "note": "only 'batch size warmup' following GLM-4.5 is mentioned (Appendix A); no batch sizes disclosed" }, "precision": { "value": null, "provenance": "unknown", "note": "master training precision not stated; disclosed: INT4 QAT applied from the SFT stage (§2.4.3), FP8 used for RL rollout inference (§3.6.2), sequence-chunked output projection promotes to higher precision during loss computation (§2.4.1)" }, "parallelism": { "value": "interleaved pipeline parallelism with flexible MTP placement (MTP output layer co-located with main output layer for parameter sharing), Pipeline ZeRO2 gradient sharding with double-buffered accumulation buffers, zero-redundant all-gather for the Muon optimizer, layer-granular pipeline activation offloading to host memory, sequence-chunked output projection, workload-aware sequence reordering with dynamic redistribution and flexible context-parallel groups using hierarchical all-to-all", "provenance": "stated", "ref": "§2.4.1, §2.4.2" }, "hardware": { "value": null, "provenance": "unknown", "note": "training hardware not disclosed (H800 appears only in a roofline discussion of DeepSeek-V3's head count, §2.1; §5 covers Chinese-chip deployment, not training)" }, "compute": { "value": null, "provenance": "unknown", "note": "no FLOPs or GPU-hours disclosed" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§3.1 (multi-task SFT over General Chat, Reasoning, and Coding & Agent categories; INT4 QAT from the SFT stage; max context 202,752 tokens; interleaved/preserved/turn-level thinking modes)" }, "preference_optimization": { "value": "GRPO", "provenance": "stated", "ref": "§3.2 ('Our RL algorithm builds upon GRPO [40] and incorporates the IcePop technique', group-normalized advantages, PPO-style clipped importance ratios, β=2, ϵlow=0.2, ϵhigh=0.28); §4.1 (agentic RL uses a group-wise policy optimization objective over K sampled traces)" }, "reasoning_training": { "value": "sequential RL pipeline: Reasoning RL (mixed domains: mathematics, science, code, tool-integrated reasoning with domain-specific judges), asynchronous Agentic RL (decoupled rollout engines via a Multi-Task Rollout Orchestrator, TITO gateway, direct double-sided importance sampling, DP-aware routing, >10K verifiable SWE/terminal/search environments), and General RL (foundational correctness, emotional intelligence, task-specific quality with hybrid rule-based/ORM/GRM rewards and human-authored anchors); on-policy cross-stage distillation as final stage", "provenance": "stated", "ref": "§3.2, §3.3, §3.4, §3.5, §4" }, "distillation": { "value": "on-policy cross-stage distillation: final checkpoints of preceding SFT/RL stages serve as teachers, with the advantage term replaced by the log-ratio between teacher and student policies (group size 1, batch 1024)", "provenance": "stated", "ref": "§3.5" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "no non-text modality described anywhere in the report; all evaluations (ARC benchmarks, CC-Bench-V2, general abilities: translation, dialogue, instruction following, world knowledge, tool calling) are text/code/tool-based" }, "attachment": { "value": null, "provenance": "n/a", "ref": "§1, §6 (text-only model)" } }, "efficiency": { "kv_cache": { "value": "MLA low-rank latent KV cache (576-dim per token: KV LoRA 512 + 64); DSA lightning indexer retrieves top-k = 2048 KV entries per query so attention is computed sparsely over cached entries; multi-node DP-attention used in RL rollouts to avoid copying KV across ranks", "provenance": "stated", "ref": "§2.1, §3.2, §3.6.2" }, "quantization": { "value": "INT4 QAT from the SFT stage with bitwise-identical training/inference kernel (§2.4.3); on Ascend: W4A8 mixed-precision (MoE experts W4A8/INT4, attention and MLP blocks W8A8/INT8, QuaRot + Flex_AWQ_SSZ calibration) fitting the 750B model on one Atlas 800T A3 (§5); FP8 for RL rollout inference (§3.6.2)", "provenance": "stated", "ref": "§2.4.3, §5, §3.6.2" }, "speculative_dedup": { "value": "Multi-Token Prediction with parameter sharing: 3 MTP layers share parameters during training (memory cost of a single MTP layer, matching DeepSeek-V3) while increasing acceptance rate; accept length 2.76 vs DeepSeek-V3.2's 2.55 at 4 speculative steps; MTP also accelerates RL rollouts", "provenance": "stated", "ref": "§2.1 (Table 2), §3.6.2" }, "serving": { "value": "slime RL infrastructure (mixed-precision rollouts, MTP, Prefill-Decode disaggregation, no-queue multi-node serving with DP-attention, heartbeat-driven fault tolerance); DP-aware routing with consistent hashing for KV-cache locality; on Ascend: fused Lightning Indexer / Sparse Flash Attention / MLAPO kernels, vLLM-Ascend and SGLang with asynchronous scheduling, RadixCache and prefix cache, FlashComm", "provenance": "stated", "ref": "§3.6, §4.1.2, §5" } }, "evaluation": { "benchmarks": [ { "name": { "value": "HLE", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 30.5, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "HLE (w/ Tools)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 50.4, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "AIME 2026 I", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 92.7, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "HMMT Feb. 2025", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 97.9, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "IMO-AnswerBench", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 82.5, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 86.0, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "LongBench v2", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 64.5, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "SWE-bench Verified", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 77.8, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "SWE-bench Multilingual", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 73.3, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Terminal-Bench 2.0 (Terminus-2)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": "56.2 / 60.7", "provenance": "stated", "ref": "Table 7 (60.7† on a verified version)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "CyberGym", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 43.2, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "BrowseComp (w/ Context Manage)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 75.9, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "BrowseComp-ZH", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 72.7, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "τ2-Bench", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 89.7, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "MCP-Atlas (Public Set)", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 67.8, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Tool-Decathlon", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 39.2, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Vending-Bench 2", "provenance": "stated", "ref": "Table 7" }, "value": { "value": "$4,432", "provenance": "stated", "ref": "Table 7 (final account balance)" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "GDPval-AA Elo", "provenance": "stated", "ref": "Table 7" }, "value": { "value": 1409, "provenance": "stated", "ref": "Table 7" }, "ref": { "value": "Table 7", "provenance": "stated", "ref": "Table 7" } }, { "name": { "value": "Artificial Analysis Intelligence Index v4.0", "provenance": "stated", "ref": "§1 (Figure 2)" }, "value": { "value": 50, "provenance": "stated", "ref": "§1 ('GLM-5 scores 50 on the Intelligence Index v4.0 and is the new open weights leader')" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } }, { "name": { "value": "SWE-rebench (Resolved Rate)", "provenance": "stated", "ref": "Table 9 (January 2026)" }, "value": { "value": 42.1, "provenance": "stated", "ref": "Table 9" }, "ref": { "value": "Table 9", "provenance": "stated", "ref": "Table 9" } }, { "name": { "value": "CC-Bench-V2 Backend Engineering (Pass@1)", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 25.8, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } }, { "name": { "value": "CC-Bench-V2 Long-horizon Chained Tasks (Pass@1)", "provenance": "stated", "ref": "Table 8" }, "value": { "value": 52.3, "provenance": "stated", "ref": "Table 8" }, "ref": { "value": "Table 8", "provenance": "stated", "ref": "Table 8" } } ] }, "contributions": { "claimed": [ { "text": { "value": "Adoption of DeepSeek Sparse Attention (DSA): content-based top-k sparse attention (lightning indexer, k=2048) applied to all layers, cutting attention compute ~1.5-2× at long sequences with no quality degradation, enabling scaling to 744B parameters (40B active) and a 28.5T-token budget", "provenance": "stated", "ref": "§1, §2.1.1" }, "ref": { "value": "§1, §2.1.1", "provenance": "stated", "ref": "§1, §2.1.1" } }, { "text": { "value": "Muon Split: per-head matrix orthogonalization for the Muon optimizer that makes MLA match GQA-8 performance and keeps attention-logit scale stable without clipping; plus MLA-256 (QK head dim 192 → V head dim 256, heads reduced by 1/3) to cut decoding cost", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Multi-Token Prediction with parameter sharing: 3 MTP layers share parameters during training, keeping draft-model memory at DeepSeek-V3 level while raising acceptance length to 2.76 (vs DeepSeek-V3.2's 2.55)", "provenance": "stated", "ref": "§2.1" }, "ref": { "value": "§2.1", "provenance": "stated", "ref": "§2.1" } }, { "text": { "value": "Fully asynchronous, decoupled RL infrastructure (slime-based): Multi-Task Rollout Orchestrator with server-based rollouts, TITO (Token-in-Token-out) gateway, direct double-sided importance sampling with token-level clipping, DP-aware routing for KV-cache locality, heartbeat-driven fault tolerance and PD disaggregation", "provenance": "stated", "ref": "§3.3, §3.6, §4.1" }, "ref": { "value": "§3.3, §3.6, §4.1", "provenance": "stated", "ref": "§3.3, §3.6, §4.1" } }, { "text": { "value": "Novel asynchronous agent RL algorithms that learn from diverse long-horizon interactions, with deterministic indexer top-k (torch.topk) and frozen indexer during RL to resolve DSA training-inference mismatch", "provenance": "stated", "ref": "§1, §3.2, §4.1" }, "ref": { "value": "§1, §3.2, §4.1", "provenance": "stated", "ref": "§1, §3.2, §4.1" } }, { "text": { "value": "On-policy cross-stage distillation to prevent catastrophic forgetting across the sequential Reasoning → Agentic → General RL pipeline", "provenance": "stated", "ref": "§3.5" }, "ref": { "value": "§3.5", "provenance": "stated", "ref": "§3.5" } }, { "text": { "value": "CC-Bench-V2: an automated internal evaluation suite (frontend/backend/long-horizon) with Agent-as-a-Judge, removing human labeling entirely", "provenance": "stated", "ref": "§6.2" }, "ref": { "value": "§6.2", "provenance": "stated", "ref": "§6.2" } }, { "text": { "value": "Full-stack adaptation to seven Chinese GPU ecosystems (Huawei Ascend, Moore Threads, Hygon, Cambricon, Kunlunxin, MetaX, Enflame) from kernels to inference frameworks, with W4A8 quantization fitting the model on a single Atlas 800T A3 machine", "provenance": "stated", "ref": "§1, §5" }, "ref": { "value": "§1, §5", "provenance": "stated", "ref": "§1, §5" } }, { "text": { "value": "First open-weights model to score 50 on the Artificial Analysis Intelligence Index v4.0, and #1 open model on LMArena Text and Code arenas; ~20% average improvement over GLM-4.7 on the eight ARC benchmarks", "provenance": "stated", "ref": "§1" }, "ref": { "value": "§1", "provenance": "stated", "ref": "§1" } } ], "assessment": { "value": "The block is a conservative evolution of the GLM-4.5/DeepSeek-V3 MLA-MoE lineage: the same 256-total/8-active/1-shared expert organization as DeepSeek-V3, MLA retained but re-tuned (MLA-256 with a Q LoRA added, Muon Split for head-wise optimizer scaling), and the dense-attention backbone replaced by DeepSeek-V3.2's DSA sparse attention on every layer. Against atlas peers: vs deepseek-v3 the differences are DSA, Muon (not AdamW), MTP parameter sharing, and the 3-dense-layer structure; vs zai-glm-4-5 it roughly doubles parameters (355B→744B) while shrinking depth (92→78 layers) and swapping in DSA. None of the attention-side ingredients is original to this report (MLA and DSA are both credited to DeepSeek), and the sparse-attention RL-stability fix (deterministic top-k, frozen indexer) is a pragmatic engineering contribution rather than a new mechanism. The genuinely distinctive contributions are in the post-training and serving systems: fully asynchronous agentic RL with TITO gateways and direct double-sided importance sampling, on-policy cross-stage distillation, and the multi-vendor Chinese-GPU inference stack. Reported gains ('~20% over GLM-4.7', '50 on AA Index v4.0') are internal or third-party leaderboard claims and not independently verifiable from the report.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, zai-glm-4-5, zai-glm-4-5-air, moonshot-kimi-k3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "75/78 backbone layers (96.2%) are MoE layers (3 dense layers, Table 10)" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "DSA sparse attention on all layers: quadratic attention replaced by lightning-indexer top-k (k=2048) retrieval over the KV cache (§2.1.1, §3.2); no dense-attention or sliding-window layers" }, "modality": { "value": "text-only", "provenance": "derived", "note": "from modality.type (inferred: no non-text modality described)" }, "openness": { "value": "open-weights", "provenance": "derived", "note": "open_weights stated true; data composition described (§2.2, §2.3) but the corpus itself is not released" }, "scale_class": { "value": "frontier", "provenance": "derived", "note": "744B ≥ 100B" }, "design_intent": { "value": "frontier-generalist", "provenance": "inferred", "note": "framed as a next-generation flagship foundation model spanning agentic, reasoning, and coding (ARC) plus general abilities (translation, dialogue, instruction following, world knowledge, tool calling); agentic/reasoning emphasis noted in prose" } }, "prose": { "design_overview": { "value": "GLM-5 is a 78-layer (3 dense + 75 MoE), 744B-parameter sparse MoE with 40B active parameters and 8 of 256 experts (1 shared) active per token. Its defining move is replacing the dense attention of GLM-4.5 with DeepSeek Sparse Attention on every layer: a lightning indexer (32 heads, head dim 128) scores all cached KV entries per query and attention is computed only over the top-k = 2048 retrieved tokens, cutting long-sequence attention cost ~1.5-2×. The attention itself remains DeepSeek-style MLA, re-tuned as MLA-256 (64 heads, QK head dim 192, V head dim 256, 576-dim latent KV) with an added Q LoRA, and with Muon Split — per-head orthogonalization of the attention up-projections — stabilizing logit scale without clipping. Decoding is sped by sharing parameters across 3 MTP layers, which also serve as the speculative draft. Context grows 4K → 200K through a staged mid-training curriculum (32K/128K/200K) with long-context agentic data, plus a DSA warm-up and 20B-token sparse-adaptation stage; SFT then runs at 202,752-token windows. Post-training is a sequential Reasoning → Agentic → General RL pipeline (GRPO-based with IcePop-style mismatch control; asynchronous, decoupled rollout engines with a TITO gateway and direct double-sided importance sampling for agentic work) closed by on-policy cross-stage distillation. The report's framing is 'vibe coding to agentic engineering': long-horizon SWE, terminal, search and slide-generation environments, CC-Bench-V2 evaluation, and a full-stack adaptation to seven Chinese GPU platforms with W4A8 quantization. Note: the gallery's GLM-5.1 and GLM-5.2 cards cite this report, but the report itself discloses only GLM-5.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Multi-head Latent Attention (MLA) — latent KV compression retained in the MLA-256 variant", "provenance": "stated", "ref": "§2.1 (cites DeepSeek-V2 [24])" }, "origin": { "value": "DeepSeek-V2", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "DeepSeek Sparse Attention (DSA) — content-based top-k sparse attention via a lightning indexer", "provenance": "stated", "ref": "§2.1.1 (cites DeepSeek-V3.2 [9])" }, "origin": { "value": "DeepSeek-V3.2", "provenance": "stated", "ref": "§2.1.1" } }, { "mechanism": { "value": "Multi-Token Prediction (MTP) with a single (here parameter-shared) MTP layer as speculative draft", "provenance": "stated", "ref": "§2.1 (cites DeepSeek-V3 [25])" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§2.1" } }, { "mechanism": { "value": "Muon optimizer recipe (per-head 'Muon Split' orthogonalization is the GLM-5 adaptation of the GLM-4.5 recipe)", "provenance": "stated", "ref": "§2.1, Appendix A (follows GLM-4.5 settings)" }, "origin": { "value": "GLM-4.5", "provenance": "stated", "ref": "§2.1, Appendix A" } }, { "mechanism": { "value": "slime framework (decoupled rollout engines initialized in GLM-4.5) reused as the unified RL infrastructure", "provenance": "stated", "ref": "§1, §3.6" }, "origin": { "value": "GLM-4.5", "provenance": "stated", "ref": "§1, §3.6" } }, { "mechanism": { "value": "IcePop technique for mitigating training-inference mismatch in RL (removing its KL term)", "provenance": "stated", "ref": "§3.2 (cites IcePop [61])" }, "origin": { "value": "IcePop (not in atlas)", "provenance": "stated", "ref": "§3.2" } } ], "influence_out": [], "notable_omissions": { "value": [ "GLM-5.1 and GLM-5.2 are never mentioned in this report (the gallery lists cards for them citing this same report)", "Positional encoding never named (no RoPE/NoPE statement; the 576-dim MLA latent hints at a 64-dim RoPE component but it is unstated)", "Tokenizer algorithm not described (only the 154,880 vocab size in Table 10)", "License for the weight release not named", "Router function (gating) and MoE load balancing not described", "Normalization type/placement and activation function not stated", "Master training precision not stated (only INT4 QAT at SFT and FP8 rollouts)", "Batch sizes and peak pre-training learning-rate values not printed (deferred to GLM-4.5 settings)", "Training hardware and total compute (FLOPs/GPU-hours) not disclosed", "No pretraining data-mixture percentages disclosed", "Input/output embedding tying not disclosed", "The prose claim of 'layer count 80' (§2.1) is inconsistent with Table 10's 3 dense + 75 MoE = 78 backbone layers and is never reconciled" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "zai-glm-5.json" }, { "slug": { "value": "zyphra-zaya1-8b", "provenance": "stated", "ref": "pipeline" }, "analysed_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "skill_version": { "value": "v0.3.5", "provenance": "stated", "ref": "pipeline" }, "source": { "url": { "value": "https://arxiv.org/abs/2605.05365", "provenance": "stated", "ref": "pipeline" }, "fetched_date": { "value": "2026-08-10", "provenance": "stated", "ref": "pipeline" }, "stored_path": { "value": "data/sources/zyphra-zaya1-8b/", "provenance": "stated", "ref": "pipeline" } }, "identity": { "family": { "value": "ZAYA1", "provenance": "stated", "ref": "Title, §VII ('ZAYA1-8B, the first and smallest model in the ZAYA-1 family')" }, "variants": [], "org": { "value": "Zyphra", "provenance": "stated", "ref": "Title page byline (Zyphra, San Francisco, CA)" }, "release_date": { "value": "2026-05-06", "provenance": "stated", "ref": "Header: 'arXiv:2605.05365v1 [cs.AI] 06 May 2026'" }, "report_url": { "value": "https://arxiv.org/abs/2605.05365", "provenance": "stated", "ref": "pipeline" }, "license": { "value": null, "provenance": "unknown", "note": "Report never names a license (arXiv page's CC BY 4.0 covers the paper only)" }, "open_weights": { "value": null, "provenance": "unknown", "note": "Report never states that weights are released; 'release' appears only as 'this release'/'release scope' (e.g. §IV-B6) without an open-weights statement" } }, "scale": { "total_params": { "value": 8.4, "provenance": "stated", "ref": "Table I (Total parameters 8.4B)" }, "active_params_per_token": { "value": 0.76, "provenance": "stated", "ref": "Table I (Active parameters 0.76B)", "note": "Release convention rounds to 0.7B active / 8B total (Table I caption); §VII-E states 'roughly 700M active parameters' per token" }, "layers": { "value": 40, "provenance": "stated", "ref": "Table I (Transformer layers 40); Appendix D (40 MoE layers)" }, "hidden_dim": { "value": 2048, "provenance": "stated", "ref": "Table I (Hidden dimension 2048)" }, "ffn_inner_dim": { "value": 4096, "provenance": "stated", "ref": "Table I (Expert FFN width: 4096 pre-activation / 2048 post-activation)", "note": "Per-expert pre-activation width (post-activation width is 2048)" }, "ffn_ratio": { "value": 2.0, "provenance": "derived", "note": "4096 (pre-activation expert width) / 2048 (hidden dim) = 2.0; report also states 'a hidden-dimension expansion factor of 2' (§II-A3)" }, "attention_heads_q": { "value": 8, "provenance": "stated", "ref": "Table I (CCA query heads 8)" }, "attention_heads_kv": { "value": 2, "provenance": "stated", "ref": "Table I (KV heads 2); Appendix C ('CCGQA with 2 KV heads for 8 query heads')" }, "head_dim": { "value": 128, "provenance": "stated", "ref": "Table I (Head dimension 128)", "note": "Per-head dimension in the compressed latent space under 2× query compression (hidden 2048 / 8 heads = 256 uncompressed, so the schema's hidden/heads derivation does not apply)" }, "vocab_size": { "value": 262272, "provenance": "stated", "ref": "Table I (Tokenizer: Gemma3 tokenizer, 262,272 vocabulary size)" }, "embedding_tied": { "value": null, "provenance": "unknown", "note": "Not disclosed (word embedding and LM head are both optimized with AdamW in RL, §IV-E, but tying is not stated)" }, "variants": [] }, "block": { "block_type": { "value": "sparse-MoE", "provenance": "derived", "note": "All 40 layers carry a 16-expert MoE FFN (Table I; Appendix D tabulates 'MoE layers 40')" }, "moe": { "expert_count": { "value": 16, "provenance": "stated", "ref": "Table I (Experts per MoE layer 16)" }, "experts_per_token": { "value": 1, "provenance": "stated", "ref": "Table I (Routing: Top-1, no residual expert); §II-A2 (k=1)" }, "shared_experts": { "value": 0, "provenance": "stated", "ref": "§II-A3 ('we trained with top-k equal to 1 and without residual experts')" }, "routing": { "value": "ZAYA1 router: down-projection to 256 dims, Exponential Depth Averaging blend with the previous layer's routing representation, then softmax(MLP(RMSNorm(r))) scores plus learned bias b; top-1 selection via argmax_e(s_e + b_e)", "provenance": "stated", "ref": "§II-A2, Eqs. 1–4" }, "load_balancing": { "value": "Auxiliary-loss-free PID-controller-style bias balancing: bias gradient ∇b_e = p_e − 1/E (empirical routing fraction minus uniform), updated with AdamW across the global batch; builds on DeepSeek-V3's bias scheme", "provenance": "stated", "ref": "§II-A2, Eq. 5 (cites DeepSeek-AI 2025a)" }, "expert_granularity": { "value": "Relatively fine-grained: 16 experts per layer with a 2× hidden expansion (expert FFN 4096 pre-activation / 2048 post-activation)", "provenance": "stated", "ref": "§II-A3, Table I" } }, "attention_variant": { "value": "GQA", "provenance": "stated", "ref": "Table I (Attention variant: CCGQA with CCA preconditioner); Appendix C", "note": "CCGQA: GQA-mode Compressed Convolutional Attention — softmax attention performed in a compressed latent space (2× query compression) with 8 query heads sharing 2 KV heads (8× KV-cache compression vs full MHA); lightweight convolutional preconditioners precede attention" }, "attention_layer_pattern": { "value": "uniform — all 40 layers use CCGQA (compressed latent-space attention); no full-attention or SSM layers anywhere in the stack", "provenance": "stated", "ref": "§II-A ('ZAYA1-8B uses Compressed Convolutional Attention (CCA) for the attention block'), Table I, Appendix C" }, "depth_mixing": { "value": "sequential-residual", "provenance": "inferred", "note": "Absence rule: standard pre-norm residual connections with learned per-channel residual scaling (αx + β on residual stream and layer output, Eq. 6); no attention over prior layer outputs" }, "position_encoding": { "method": { "value": "RoPE", "provenance": "stated", "ref": "§II-A3 ('We applied RoPE (Su et al., 2023) to half the channels in each head, leaving the other half without position embeddings')" }, "rope_base": { "value": 5000000, "provenance": "stated", "ref": "Table II", "note": "Final training phase (131K SFT) uses RoPE base 5M; progression: 10K (4K base pretraining) → 1M (32K midtraining) → 5M (131K SFT)" }, "partial_rope": { "value": true, "provenance": "stated", "ref": "Table I (Position embeddings: 50% RoPE on each head); §II-A3" }, "extension": { "method": { "value": "other", "provenance": "stated", "note": "Context extension by raising the RoPE base frequency per training phase (10K → 1M at 32K midtraining → 5M at 131K SFT); no interpolation factor and no named method (YaRN/NTK/PI) disclosed", "ref": "Table II, §III" }, "factor": { "value": null, "provenance": "n/a", "ref": "Table II (no rescaling factor disclosed; extension via base-frequency increase)" }, "target_length": { "value": 131072, "provenance": "stated", "ref": "Table II (SFT phase at 131K context)" } } }, "normalization": { "type": { "value": "RMSNorm", "provenance": "stated", "ref": "§II-A2 Eq. 3 (router: RMSnorm(r_l)); §II-A3 Eq. 6 (Layer(RMSnorm(x_l))); Appendix C (RMSNorm head-wise temperature on keys)" }, "placement": { "value": "pre", "provenance": "stated", "ref": "§II-A3 Eq. 6 (x_{l+1} = Res-scale_res(x_l) + Res-scale_out(Layer(RMSnorm(x_l))))" }, "qk_norm": { "value": true, "provenance": "stated", "ref": "Appendix C ('The scaled query-key inner product is then upper bounded (assuming QK norm) to T·d_h/√d_h = T√d_h'); §IV-D (QK-norm and QK-mean in the FP32 op set)", "note": "CCA is softmax attention with QK-normalisation: RMSNorm applies a learned head-wise temperature strictly to the keys, query/key mean skip connections enforce representational similarity; logits bounded by T√d_h. This is QK-norm on attention layers, not a linear-attention L2 norm." } }, "activation": { "value": null, "provenance": "unknown", "note": "Report names GeLU only for the router MLP (§II-A2); the expert FFN activation is never stated (Appendix D implies a gated FFN with fused gate/up projections, 'linear_fc1', but names no activation)" }, "stability": { "attention_sinks": { "value": false, "provenance": "inferred", "note": "Absence rule: no designed sink tokens described" }, "softcapping": { "value": false, "provenance": "inferred", "note": "No logit softcapping; CCA instead bounds attention logits via QK-norm plus a learned key temperature parameterized linearly as T rather than exp(T) (Appendix C)" }, "other": { "value": [ "Learned key temperature T scaled linearly (not exp(T)) to bound the maximum attention logit at T·√d_h under QK-norm (stability modification of standard CCA)", "Query/key mean skip connections enforcing representational similarity", "Learned residual scaling (α, β per layer on residual stream and layer output) controlling residual-norm growth through depth, with α initialized to ones and β to zeros" ], "provenance": "stated", "ref": "Appendix C, §II-A3 Eq. 6" } } }, "context": { "trained_length": { "value": 131072, "provenance": "stated", "ref": "Table II (SFT phase at 131K context, 660B tokens, RoPE base 5M)", "note": "Earlier stages: 4K base pretraining (two phases), 32K midtraining (1.2T tokens)" }, "deployed_length": { "value": 131072, "provenance": "inferred", "note": "Final checkpoint is the 131K-trained SFT+RL model; the report prints no separate deployed length, and RL rollouts run up to 81,920-token responses within the 131K context" }, "extension_method": { "value": "Staged RoPE base-frequency increase: 10K (4K context) → 1M (32K midtraining) → 5M (131K SFT); no interpolation factor stated", "provenance": "stated", "ref": "Table II, §III" } }, "tokenizer": { "algorithm": { "value": null, "provenance": "unknown", "note": "Report names the Gemma3 tokenizer with 262,272 vocabulary size (Table I) but never describes its algorithm" }, "notes": { "value": "Gemma3 tokenizer, 262,272 vocabulary size", "provenance": "stated", "ref": "Table I" } }, "training": { "tokens": { "value": 13.86, "provenance": "derived", "note": "8T (base pretraining phase 1) + 4T (phase 2) + 1.2T (32K midtraining) + 0.66T (131K SFT) = 13.86T; per-phase budgets from Table II (includes midtraining and SFT, which the report counts as training phases)" }, "data_composition": { "value": "Base pretraining: broad web-crawl distribution with code, math, multilingual, and reasoning data mixed in progressively; phase 2 upweights code, math, reasoning, and instruction-formatted data. 32K midtraining mixture: 86.1% long-CoT reasoning traces, 5.7% web/synthetic web/multilingual, 0.8% natively long-context, 3.0% code corpus, 3.0% math/STEM, 1.4% short instruction/few-shot. 131K SFT mixture: 75.0% long-CoT, 9.8% web, 6.4% natively long-context, 5.0% code corpus/code SFT, 2.6% math/STEM, 1.2% short instruction. Individual dataset names are omitted.", "provenance": "stated", "ref": "§III, Tables II–III" }, "curriculum": { "value": "Staged: 4K base pretraining (two phases) → 32K reasoning-focused midtraining (1.2T tokens) → 131K SFT (660B tokens); long-CoT reasoning data in every pretraining/midtraining phase (majority of the midtrain mix) with stage-aware answer-preserving (AP) re-trimming at each context length; RL cascade follows post-training", "provenance": "stated", "ref": "§III, §III-A" }, "optimizer": { "value": "Muon with AdamW RMS matching (pretraining, midtraining, SFT); RL: momentum-free Muon for matrix-valued actor weights with AdamW for embeddings and LM head", "provenance": "stated", "ref": "§III, §IV-E" }, "lr_schedule": { "value": null, "provenance": "unknown", "note": "No schedule described; RL learning rates are per-stage in the range 2×10⁻⁶ to 1×10⁻⁵ (§IV-B1)" }, "batch_schedule": { "value": "RL: minibatches of 128 prompts with rollout group size G=16, 131,072-token budget per GPU per microbatch with sequence packing and dynamic batching; pretraining run example (Appendix B): G=4096, s=4096, 4 bytes/token", "provenance": "stated", "ref": "§IV-B1, §IV-C, Appendix B" }, "precision": { "value": "BF16 weights and activations with a matched FP32 op set on trainer and inference engine: fused cross-entropy + LM-head matmul, CCA cache state, QK-norm, QK-mean, RMSNorm, router softmax, residual-stream additions", "provenance": "stated", "ref": "§IV-D" }, "parallelism": { "value": "Data parallelism plus all-gather KV context parallelism (2 ranks at 32K, 8 ranks at 131K); FSDP2 shard size 4 with sequence parallelism disabled; post-training uses asynchronous PipelineRL with disjoint rollout/trainer GPU pools (2–5× more rollout workers, weight sync every 2 trainer iterations)", "provenance": "stated", "ref": "§III, §IV-C, §IV-B1" }, "hardware": { "value": "AMD MI300X GPUs (8× per node, Infinity Fabric intra-node) with AMD Pensando Pollara 400 networking (400 Gb/s NICs); dual-socket Intel Xeon Platinum 8570 nodes; 25.6 TB NVMe storage per compute node", "provenance": "stated", "ref": "Table I, Appendix A (Table XIV)" }, "compute": { "value": null, "provenance": "unknown", "note": "No FLOPs or GPU-hours disclosed; token budgets per phase are given (Table II)" } }, "post_training": { "sft": { "value": true, "provenance": "stated", "ref": "§IV-A ('SFT phase at 131K context for 660B tokens'; chat, IF, code, math, reasoning, tool-calling and TTC-aggregation mixture; best-fit decreasing bin packing)" }, "preference_optimization": { "value": "other", "provenance": "stated", "ref": "§IV-B, §IV-B1, §IV-B7", "note": "Four-stage RL cascade (reasoning warmup → RLVE-Gym adaptive curriculum → math+code+TTC in two phases → behavioral RL) on a shared spine: DPPO Binary-TV trust-region masking (δ=0.1), Dr-GRPO SMTSN loss aggregation, MaxRL advantage estimation (per-prompt mean normalization); final behavioral stage uses standard GRPO with reward-model scores gated by a binary IF checker and is described as 'behavioral RLHF'; no KL regularization in the reward" }, "reasoning_training": { "value": "Trained from scratch for reasoning: long-CoT data in all pretraining/midtraining phases with answer-preserving trimming; verifiable RL over math and puzzles (232-step warmup; 400-task RLVE-Gym adaptive difficulty curriculum with Thompson-sampling IRT calibration), code (CodeI/O prediction, CodeARC reconstruction, falsification synthetic environments built from competitive-programming references), and TTC aggregation prompts (Markovian RSA expert- and self-aggregation); per-rollout response budgets up to 81,920 tokens; no dedicated multi-turn agentic RL stage", "provenance": "stated", "ref": "Abstract, §III-A, §IV-B3–B6, §VI-B" }, "distillation": { "value": "none", "provenance": "stated", "ref": "Abstract ('ZAYA1-8B was trained from scratch for reasoning')", "note": "Teacher-model rollouts are reused to construct Markovian RSA aggregation SFT examples (§VI-B) — teacher-generated training data, not distillation" } }, "modality": { "type": { "value": "text-only", "provenance": "inferred", "note": "Every described capability, training-data category, and benchmark is text-based (math, code, chat, instruction following); no vision/audio modality is mentioned anywhere" }, "attachment": { "value": null, "provenance": "n/a", "ref": "modality.type text-only" } }, "efficiency": { "kv_cache": { "value": "CCGQA: KV-cache compression 8× relative to full multi-head attention (2 KV heads shared across 8 query heads on top of 2× query compression); CCA's compressed latent KV representation kept activation and KV-cache memory low enough for 131K-context SFT and RL", "provenance": "stated", "ref": "Table I, §III, Appendix C" }, "quantization": { "value": "none disclosed", "provenance": "stated", "ref": "report" }, "speculative_dedup": { "value": "none disclosed", "provenance": "stated", "ref": "report" }, "serving": { "value": "none disclosed", "provenance": "stated", "ref": "report", "note": "The Markovian RSA serving profile (bounded-context batched decoding, ≈0.4× wall-clock of long-reasoning baseline) is described in §VI-C but no serving stack/optimizations are disclosed" } }, "evaluation": { "benchmarks": [ { "name": { "value": "AIME'26", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 89.1, "provenance": "stated", "ref": "Table VII (avg@64)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "HMMT'26 Feb.", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 71.6, "provenance": "stated", "ref": "Table VII (avg@64)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "IMO-AnswerBench", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 59.3, "provenance": "stated", "ref": "Table VII (avg@64)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "APEX-shortlist", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 32.2, "provenance": "stated", "ref": "Table VII (avg@64)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "LiveCodeBench-v6", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 64.8, "provenance": "stated", "ref": "Table VII (avg@16)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "GPQA-Diamond", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 71.0, "provenance": "stated", "ref": "Table VII (avg@16)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "MMLU-Pro", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 74.2, "provenance": "stated", "ref": "Table VII (mean@1)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "IFEval", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 85.6, "provenance": "stated", "ref": "Table VII (mean@1)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "IFBench", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 52.6, "provenance": "stated", "ref": "Table VII (mean@1)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "EQBench", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 73.0, "provenance": "stated", "ref": "Table VII (mean@1, claude-3.7-sonnet judge)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "Creative Writing v3", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 63.0, "provenance": "stated", "ref": "Table VII (mean@1, claude-3.7-sonnet judge)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "BFCL-v4", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 40.5, "provenance": "stated", "ref": "Table VII" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "tau2", "provenance": "stated", "ref": "Table VII" }, "value": { "value": 36.3, "provenance": "stated", "ref": "Table VII (avg@16)" }, "ref": { "value": "Table VII", "provenance": "stated", "ref": "Table VII" } }, { "name": { "value": "AIME'25 (Markovian RSA 40K/4K)", "provenance": "stated", "ref": "Table XI, §VI-C" }, "value": { "value": 91.9, "provenance": "stated", "ref": "Table XI, §VI-C (headline TTC configuration β=40K, τ=4K, T=2, N=16, C=4; single-rollout AIME'25 is 88.3)" }, "ref": { "value": "Table XI, §VI-C", "provenance": "stated", "ref": "Table XI, §VI-C" } }, { "name": { "value": "HMMT'25 Feb. (Markovian RSA 40K/4K)", "provenance": "stated", "ref": "Table XI, §VI-C" }, "value": { "value": 89.6, "provenance": "stated", "ref": "Table XI, §VI-C (single-rollout HMMT'25 is 82.7)" }, "ref": { "value": "Table XI, §VI-C", "provenance": "stated", "ref": "Table XI, §VI-C" } } ] }, "contributions": { "claimed": [ { "text": { "value": "CCA (Compressed Convolutional Attention) validated at 8B scale: sequence mixing in a compressed latent space via low-rank projections and convolutional preconditioners, competitive with MLA/GQA while cutting prefill FLOPs and KV-cache memory (8× compression in CCGQA mode)", "provenance": "stated", "ref": "§I, §II-A1, Appendix C" }, "ref": { "value": "§I, §II-A1, Appendix C", "provenance": "stated", "ref": "§I, §II-A1, Appendix C" } }, { "text": { "value": "ZAYA1 router: MLP-based router with Exponential Depth Averaging and PID-controller-style bias balancing, substantially more expressive than linear routers; better routing decisions reduce balancing instability and improve quality", "provenance": "stated", "ref": "§I, §II-A2" }, "ref": { "value": "§I, §II-A2", "provenance": "stated", "ref": "§I, §II-A2" } }, { "text": { "value": "Learned residual scaling (per-channel α, β on the residual stream and layer input at each block) controlling residual-norm growth through depth at negligible parameter/FLOP cost", "provenance": "stated", "ref": "§I, §II-A3" }, "ref": { "value": "§I, §II-A3", "provenance": "stated", "ref": "§I, §II-A3" } }, { "text": { "value": "Answer-preserving (AP) trimming: training-data construction that truncates the tail of over-length reasoning traces while preserving the final answer, enabling long-CoT data in short-context pretraining", "provenance": "stated", "ref": "§I, §III-A" }, "ref": { "value": "§I, §III-A", "provenance": "stated", "ref": "§I, §III-A" } }, { "text": { "value": "Four-stage cascaded RL pipeline (reasoning warmup, 400-task RLVE-Gym adaptive curriculum, math+code+TTC with synthetic code environments, behavioral RL) with DPPO Binary-TV trust region, Dr-GRPO aggregation, and MaxRL advantages", "provenance": "stated", "ref": "§I, §IV-B" }, "ref": { "value": "§I, §IV-B", "provenance": "stated", "ref": "§I, §IV-B" } }, { "text": { "value": "Markovian RSA: a test-time compute method recursively aggregating parallel reasoning traces while carrying forward only bounded-length tails (τ of β), trained into the model via SFT aggregation examples and RL variants; 91.9% AIME'25 / 89.6% HMMT'25 at β=40K/τ=4K", "provenance": "stated", "ref": "§I, §VI" }, "ref": { "value": "§I, §VI", "provenance": "stated", "ref": "§I, §VI" } }, { "text": { "value": "Evidence that a full-stack AMD platform (MI300X + Pensando Pollara 400) can support sustained pretraining, long-context midtraining, and SFT for an 8B-total-parameter MoE reasoning model", "provenance": "stated", "ref": "§I, §III, Appendix A" }, "ref": { "value": "§I, §III, Appendix A", "provenance": "stated", "ref": "§I, §III, Appendix A" } }, { "text": { "value": "With under 1B active parameters, ZAYA1-8B matches or exceeds DeepSeek-R1-0528 on several challenging mathematics and coding benchmarks and stays competitive with much larger open-weight reasoning models", "provenance": "stated", "ref": "Abstract, §I, Table XI" }, "ref": { "value": "Abstract, §I, Table XI", "provenance": "stated", "ref": "Abstract, §I, Table XI" } } ], "assessment": { "value": "The three architectural changes are refinements of known families rather than new mechanisms: CCA/CCGQA is softmax attention moved into a compressed latent space with convolutional preconditioning (a compute-efficiency variant of GQA — the atlas's first compressed-latent attention, but conceptually adjacent to MLA's latent KV); the ZAYA1 router upgrades DeepSeek-V3-style bias balancing (PID update via AdamW) and adds a DenseFormer-style depth-averaged MLP router; residual scaling is a cheap alternative to Qwen-style attention gating. Top-1 routing with no shared experts is an aggressive sparsity choice not seen in atlas peers (deepseek-v3/mixtral use top-8 with shared experts). The genuinely novel contribution is Markovian RSA with training-time integration — the first TTC workflow trained into the model in the atlas — though it is an inference/training scheme, not an architecture change. vs deepseek-r1: same reasoning-specialised, verifiable-RL recipe at far smaller scale. The report is exemplary on training disclosure (phases, mixtures, precision, optimizer, stability engineering) but omits the expert FFN activation, tokenizer algorithm, training compute, and any weights-release statement.", "provenance": "inferred", "note": "comparison against atlas entries (deepseek-v3, mistral-mixtral-8x7b, deepseek-r1, allenai-olmo-3, moonshot-kimi-k3) from their stored sources" } }, "taxonomy": { "compute_structure": { "value": "sparse-MoE", "provenance": "derived", "note": "40/40 layers carry 16-expert routed MoE FFNs (top-1 routing; Table I, Appendix D)" }, "sequence_mixing": { "value": "efficient-attention", "provenance": "derived", "note": "All 40 layers use CCGQA, a compressed latent-space attention variant (100% ≥ 80% layer-majority rule)" }, "modality": { "value": "text-only", "provenance": "derived", "note": "From modality.type (inferred text-only: no non-text modality described)" }, "openness": { "value": "undisclosed", "provenance": "derived", "note": "Report never states weights are released (open_weights unknown) and names no license; data mixtures described only at category level with individual datasets omitted" }, "scale_class": { "value": "medium", "provenance": "derived", "note": "8.4B total: 1B ≤ 8.4B < 10B" }, "design_intent": { "value": "reasoning-specialised", "provenance": "inferred", "note": "Framed as 'a reasoning-focused mixture-of-experts model' designed 'to maximize reasoning performance per active parameter' (§I, §VII) with math/code emphasis and TTC co-design; agentic and broad-knowledge capabilities explicitly deprioritized (§IV-B6, §VII-G)" } }, "prose": { "design_overview": { "value": "ZAYA1-8B is Zyphra's reasoning-focused MoE: 8.4B total parameters, 0.76B active, 40 layers, 16 experts per layer with top-1 routing and no shared or residual experts. Its block makes three departures from standard MoE transformers. First, sequence mixing uses Compressed Convolutional Attention in GQA mode (CCGQA): attention is performed in a compressed latent space after low-rank down-projections and lightweight sequence-mixing convolutions (a short convolution plus grouped head-wise convolution) precondition queries and keys; half the value heads are time-delayed by one token; query/key mean skip connections and an RMSNorm that applies a learned head-wise temperature to the keys (parameterized linearly as T, not exp(T)) keep attention logits bounded under QK-norm. With 8 query heads sharing 2 KV heads on top of 2× query compression, the KV cache is 8× smaller than full MHA. Second, the ZAYA1 router replaces the linear router with a down-projection to 256 dims, an Exponential Depth Averaging blend with the previous layer's routing representation (a DenseFormer-style depth-weighted averaging variant), and a three-layer GeLU MLP, with PID-controller-style bias balancing (AdamW on p_e − 1/E) rather than DeepSeek-style sign updates. Third, learned residual scaling (per-channel α, β on both the residual stream and layer outputs) controls residual-norm growth through depth. The model was trained from scratch for reasoning — long-CoT data in every pretraining phase with answer-preserving trimming, a 32K midtraining and 131K SFT on AMD MI300X + Pollara hardware, then a four-stage verifiable-RL cascade — and is co-trained with Markovian RSA, a test-time-compute workflow that recursively aggregates candidate reasoning traces while carrying forward only bounded tails (e.g. 4K of a 40K budget), lifting AIME'25 to 91.9%.", "provenance": "inferred", "note": "authored by atlas analyst" }, "lineage_in": [ { "mechanism": { "value": "Compressed Convolutional Attention (CCA) — compressed latent-space attention with convolutional preconditioners, evaluated here at 8B scale", "provenance": "stated", "ref": "§I, §II-A1, Appendix C (cites Figliolia et al. 2025)" }, "origin": { "value": "CCA (Figliolia et al., 2025) (not in atlas)", "provenance": "stated", "ref": "§I" } }, { "mechanism": { "value": "Exponential Depth Averaging (EDA) — a variant of Depth-Weighted Averaging for the router representation", "provenance": "stated", "ref": "§II-A2 (cites Pagliardini et al. 2024, DenseFormer)" }, "origin": { "value": "DenseFormer (Pagliardini et al., 2024) (not in atlas)", "provenance": "stated", "ref": "§II-A2" } }, { "mechanism": { "value": "Auxiliary-loss-free bias-based load balancing — PID-style update of the DeepSeek bias scheme", "provenance": "stated", "ref": "§II-A2 (cites DeepSeek-AI 2025a, DeepSeek-V3)" }, "origin": { "value": "DeepSeek-V3", "provenance": "stated", "ref": "§II-A2" } }, { "mechanism": { "value": "Grouped-Query Attention — KV sharing retained in CCGQA's 8-query/2-KV structure", "provenance": "stated", "ref": "§II-A1, Appendix C (cites Ainslie et al. 2023)" }, "origin": { "value": "GQA (Ainslie et al., 2023)", "provenance": "stated", "ref": "§II-A1" } }, { "mechanism": { "value": "Markovian RSA combines Recursive Self-Aggregation with the bounded-workspace principle of Markovian Thinker", "provenance": "stated", "ref": "§VI (cites Venkatraman et al. 2025; Aghajohari et al. 2025)" }, "origin": { "value": "RSA (Venkatraman et al., 2025) / Markovian Thinker (Aghajohari et al., 2025) (not in atlas)", "provenance": "stated", "ref": "§VI" } }, { "mechanism": { "value": "Muon optimizer (with AdamW RMS matching; momentum-free variant for RL)", "provenance": "stated", "ref": "§III, §IV-E (cites Jordan et al. 2024)" }, "origin": { "value": "Muon (Jordan et al., 2024) (not in atlas)", "provenance": "stated", "ref": "§III" } } ], "influence_out": [], "notable_omissions": { "value": [ "No statement that weights are released and no license named (openness undisclosed)", "Expert FFN activation never named (GeLU appears only for the router MLP); Appendix D implies a gated FFN ('linear_fc1') without naming the activation", "Tokenizer algorithm not described (only 'Gemma3 tokenizer, 262,272 vocabulary size')", "No training compute (FLOPs or GPU-hours) disclosed", "No learning-rate schedule disclosed (only per-stage RL LR ranges)", "Input/output embedding tying not disclosed", "No per-stage ablations of the RL cascade ('We do not report per-stage ablations in this release')", "Individual training dataset names omitted (category-level proportions only)", "No dedicated multi-turn agentic RL stage; agentic benchmarks (BFCL-v4, τ²) acknowledged to lag", "Controlled optimizer ablation not run (momentum-free Muon presented as an empirical recipe choice)", "Markovian RSA vs full-chain RSA comparison left for future work (only the τ=β limit is characterized analytically)" ], "provenance": "inferred", "note": "authored by atlas analyst" } }, "_file": "zyphra-zaya1-8b.json" } ] }