{ "schemaVersion": "leaderboard.v1", "schema": "schema/leaderboard.v1.json", "generatedAt": "2026-08-06T14:30:36.699Z", "benchmark": "VigilSAR Defense LLM Benchmark", "version": "0.1.0", "facts": { "benchmark": "VigilSAR Defense LLM Benchmark", "version": "0.1.0", "generatedAt": "2026-08-06T14:30:36.699Z", "sourceGeneratedAt": "2026-08-06T14:24:36.727Z", "taskCount": 300, "categories": [ "Communication", "Compliance", "Domain comprehension", "Geospatial reasoning", "ISR reasoning", "Legal and ethical reasoning", "Planning support", "Robustness and calibration", "Safety", "Source evaluation", "Sovereign deployability", "Structured reporting" ], "pgTaskCount": 217, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "scorerDirty": false, "heldoutGap": "-18.11", "heldoutNote": "34 scored; 0 red flag(s); threshold 5 points" }, "freshness": { "status": "fresh", "checkedAt": "2026-08-06T14:30:36.721Z", "currentTaskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "staleModels": [], "warnings": [] }, "columns": [ "model", "tier", "score.mean", "ci.lo", "ci.hi", "rankBand", "capability", "reliabilitySafety", "deployability", "ece", "eceCoverage", "safetyRiskIndex", "criticalCount", "heldoutGap", "costUsd", "costPerCorrect", "avgLatencyMs", "taskSetHash", "scorerGitRev", "pinned" ], "rows": [ { "model": "claude-code-cli:claude-fable-5", "source": null, "tier": "hosted-api", "score.mean": 67.77, "ci.lo": 64.93, "ci.hi": 70.79, "rankBand": "A", "capability": 67, "reliabilitySafety": 66.95, "deployability": 53.64, "ece": 0.241, "eceCoverage": 0.437, "safetyRiskIndex": 66, "criticalCount": 2, "heldoutGap": "pending", "costUsd": 25.73128, "costPerCorrect": 0.126568, "avgLatencyMs": 13805.66, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": true, "heldoutStatus": "pending", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "claude-code-cli:claude-fable-5-early-july", "source": null, "tier": "hosted-api", "score.mean": 65.37, "ci.lo": 62.18, "ci.hi": 68.54, "rankBand": "B", "capability": 66.3, "reliabilitySafety": 60.72, "deployability": 54.04, "ece": 0.235, "eceCoverage": 0.41, "safetyRiskIndex": 283, "criticalCount": 11, "heldoutGap": -5.81, "costUsd": 53.389567, "costPerCorrect": 0.27225, "avgLatencyMs": 15778.03, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "kimi-cli:kimi-k3", "source": null, "tier": "hosted-api", "score.mean": 64.65, "ci.lo": 61.63, "ci.hi": 67.7, "rankBand": "B", "capability": 66.63, "reliabilitySafety": 57.45, "deployability": 53.37, "ece": 0.22, "eceCoverage": 0.453, "safetyRiskIndex": 244, "criticalCount": 9, "heldoutGap": -10.97, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 21643.25, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.5-medium", "source": null, "tier": "hosted-api", "score.mean": 62.17, "ci.lo": 59.15, "ci.hi": 65.23, "rankBand": "C", "capability": 60.83, "reliabilitySafety": 61.16, "deployability": 50.97, "ece": 0.175, "eceCoverage": 0.27, "safetyRiskIndex": 250, "criticalCount": 10, "heldoutGap": -9.12, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 17649.11, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "claude-code-cli:claude-opus-5-high", "source": null, "tier": "hosted-api", "score.mean": 61.7, "ci.lo": 58.44, "ci.hi": 64.87, "rankBand": "C", "capability": 62.53, "reliabilitySafety": 55.84, "deployability": 53.37, "ece": 0.231, "eceCoverage": 0.547, "safetyRiskIndex": 160, "criticalCount": 5, "heldoutGap": -13.02, "costUsd": 33.672073, "costPerCorrect": 0.181902, "avgLatencyMs": 18166.68, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.5", "source": null, "tier": "hosted-api", "score.mean": 61.66, "ci.lo": 58.54, "ci.hi": 64.72, "rankBand": "C", "capability": 60.92, "reliabilitySafety": 60.38, "deployability": 49.1, "ece": 0.199, "eceCoverage": 0.283, "safetyRiskIndex": 283, "criticalCount": 11, "heldoutGap": "pending", "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 11839.06, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "pending", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.4-mini", "source": null, "tier": "hosted-api", "score.mean": 60.22, "ci.lo": 57.31, "ci.hi": 63.12, "rankBand": "C", "capability": 60.6, "reliabilitySafety": 54.95, "deployability": 52.17, "ece": 0.186, "eceCoverage": 0.337, "safetyRiskIndex": 116, "criticalCount": 4, "heldoutGap": -5.61, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 22229.95, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.6-terra", "source": null, "tier": "hosted-api", "score.mean": 58.61, "ci.lo": 55.65, "ci.hi": 61.5, "rankBand": "D", "capability": 59.79, "reliabilitySafety": 53.29, "deployability": 50.84, "ece": 0.155, "eceCoverage": 0.313, "safetyRiskIndex": 183, "criticalCount": 7, "heldoutGap": -6.56, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 6572.61, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.6-sol", "source": null, "tier": "hosted-api", "score.mean": 57.86, "ci.lo": 54.92, "ci.hi": 60.87, "rankBand": "D", "capability": 56.27, "reliabilitySafety": 56.53, "deployability": 51.64, "ece": 0.159, "eceCoverage": 0.297, "safetyRiskIndex": 233, "criticalCount": 9, "heldoutGap": -4.86, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 7373.9, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.6-terra-xhigh", "source": null, "tier": "hosted-api", "score.mean": 57.69, "ci.lo": 54.88, "ci.hi": 60.65, "rankBand": "D", "capability": 58.44, "reliabilitySafety": 51.89, "deployability": 50.84, "ece": 0.162, "eceCoverage": 0.31, "safetyRiskIndex": 158, "criticalCount": 6, "heldoutGap": -6.15, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 9838.02, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.3-codex-spark", "source": null, "tier": "hosted-api", "score.mean": 57.23, "ci.lo": 54.22, "ci.hi": 60.13, "rankBand": "D", "capability": 58.94, "reliabilitySafety": 50.08, "deployability": 50.7, "ece": 0.161, "eceCoverage": 0.37, "safetyRiskIndex": 225, "criticalCount": 9, "heldoutGap": -3.5, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 5748.69, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:thinkingmachines/inkling@default", "source": null, "tier": "hosted-api", "score.mean": 55.95, "ci.lo": 52.45, "ci.hi": 59.32, "rankBand": "D", "capability": 57.27, "reliabilitySafety": 52.72, "deployability": 44.04, "ece": 0.154, "eceCoverage": 0.323, "safetyRiskIndex": 116, "criticalCount": 4, "heldoutGap": -4.02, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 27198.14, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "codex-cli:gpt-5.6-luna", "source": null, "tier": "hosted-api", "score.mean": 55.49, "ci.lo": 52.53, "ci.hi": 58.65, "rankBand": "E", "capability": 53.61, "reliabilitySafety": 53.73, "deployability": 51.1, "ece": 0.143, "eceCoverage": 0.29, "safetyRiskIndex": 258, "criticalCount": 10, "heldoutGap": -7.54, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 7895.3, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:z-ai/glm-5.2@default", "source": null, "tier": "hosted-api", "score.mean": 54.48, "ci.lo": 51.19, "ci.hi": 57.82, "rankBand": "E", "capability": 55.45, "reliabilitySafety": 50.47, "deployability": 44.7, "ece": 0.134, "eceCoverage": 0.307, "safetyRiskIndex": 266, "criticalCount": 10, "heldoutGap": -16.04, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 151065.39, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/nemotron-3-ultra-550b-a55b@default", "source": null, "tier": "hosted-api", "score.mean": 54.39, "ci.lo": 51.25, "ci.hi": 57.58, "rankBand": "E", "capability": 58.03, "reliabilitySafety": 43.51, "deployability": 48.7, "ece": 0.134, "eceCoverage": 0.357, "safetyRiskIndex": 233, "criticalCount": 9, "heldoutGap": -6.86, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 12220.78, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:minimaxai/minimax-m3@default", "source": null, "tier": "hosted-api", "score.mean": 53.6, "ci.lo": 50.44, "ci.hi": 56.92, "rankBand": "E", "capability": 53.52, "reliabilitySafety": 49.27, "deployability": 46.44, "ece": 0.125, "eceCoverage": 0.357, "safetyRiskIndex": 233, "criticalCount": 9, "heldoutGap": -17.66, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 37999.25, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "gemini-cli:gemini-3.1-pro-preview", "source": null, "tier": "hosted-api", "score.mean": 53.03, "ci.lo": 49.92, "ci.hi": 56.22, "rankBand": "E", "capability": 55.42, "reliabilitySafety": 45.28, "deployability": 45.5, "ece": 0.133, "eceCoverage": 0.257, "safetyRiskIndex": 383, "criticalCount": 15, "heldoutGap": "pending", "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 15791.07, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "pending", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "mlx-openai:qwen/qwen3.6-35b-a3b", "source": null, "tier": "sovereign-deployable", "score.mean": 52.78, "ci.lo": 49.71, "ci.hi": 55.85, "rankBand": "E", "capability": 55.06, "reliabilitySafety": 47.18, "deployability": 68.64, "ece": 0.124, "eceCoverage": 0.32, "safetyRiskIndex": 100, "criticalCount": 4, "heldoutGap": "pending", "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 20177.4, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "pending", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:deepseek-ai/deepseek-v4-flash@default", "source": null, "tier": "hosted-api", "score.mean": 52.45, "ci.lo": 49.08, "ci.hi": 55.72, "rankBand": "E", "capability": 55.46, "reliabilitySafety": 42.02, "deployability": 48.3, "ece": 0.099, "eceCoverage": 0.29, "safetyRiskIndex": 341, "criticalCount": 13, "heldoutGap": -5.01, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 27385.04, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "gemini-cli:gemini-3.1-flash-lite", "source": null, "tier": "hosted-api", "score.mean": 51.15, "ci.lo": 47.9, "ci.hi": 54.43, "rankBand": "F", "capability": 53.76, "reliabilitySafety": 41.67, "deployability": 48.97, "ece": 0.119, "eceCoverage": 0.3, "safetyRiskIndex": 400, "criticalCount": 16, "heldoutGap": "pending", "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 7764.57, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "pending", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:qwen/qwen3.5-397b-a17b@default", "source": null, "tier": "hosted-api", "score.mean": 48.95, "ci.lo": 45.82, "ci.hi": 52.17, "rankBand": "F", "capability": 50.04, "reliabilitySafety": 47.01, "deployability": 36.3, "ece": 0.112, "eceCoverage": 0.273, "safetyRiskIndex": 33, "criticalCount": 1, "heldoutGap": -17.78, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 195031.35, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:qwen/qwen3-next-80b-a3b-instruct@default", "source": null, "tier": "hosted-api", "score.mean": 48.13, "ci.lo": 45.11, "ci.hi": 51.42, "rankBand": "G", "capability": 49.46, "reliabilitySafety": 42.82, "deployability": 43.64, "ece": 0.117, "eceCoverage": 0.277, "safetyRiskIndex": 183, "criticalCount": 7, "heldoutGap": -2.11, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 254810.55, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:mistralai/mistral-medium-3.5-128b@default", "source": null, "tier": "hosted-api", "score.mean": 47.53, "ci.lo": 44.57, "ci.hi": 50.62, "rankBand": "G", "capability": 49.39, "reliabilitySafety": 38.94, "deployability": 45.64, "ece": 0.116, "eceCoverage": 0.26, "safetyRiskIndex": 425, "criticalCount": 17, "heldoutGap": -7.14, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 214062.35, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:google/gemma-4-31b-it@default", "source": null, "tier": "hosted-api", "score.mean": 47.13, "ci.lo": 43.8, "ci.hi": 50.64, "rankBand": "G", "capability": 48.38, "reliabilitySafety": 40.45, "deployability": 44.3, "ece": 0.099, "eceCoverage": 0.257, "safetyRiskIndex": 100, "criticalCount": 4, "heldoutGap": -13.89, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 42388.12, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/nemotron-3-nano-omni-30b-a3b-reasoning@default", "source": null, "tier": "hosted-api", "score.mean": 46.88, "ci.lo": 43.61, "ci.hi": 50.1, "rankBand": "G", "capability": 49.99, "reliabilitySafety": 36.49, "deployability": 45.1, "ece": 0.127, "eceCoverage": 0.29, "safetyRiskIndex": 500, "criticalCount": 20, "heldoutGap": -5.05, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 11472.2, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:openai/gpt-oss-20b@default", "source": null, "tier": "hosted-api", "score.mean": 46.72, "ci.lo": 43.5, "ci.hi": 50.02, "rankBand": "G", "capability": 51.89, "reliabilitySafety": 34.45, "deployability": 44.84, "ece": 0.143, "eceCoverage": 0.343, "safetyRiskIndex": 316, "criticalCount": 12, "heldoutGap": -15.34, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 2699.75, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:mistralai/mistral-nemotron@default", "source": null, "tier": "hosted-api", "score.mean": 46.56, "ci.lo": 43.6, "ci.hi": 49.66, "rankBand": "G", "capability": 48.19, "reliabilitySafety": 38.2, "deployability": 47.36, "ece": 0.115, "eceCoverage": 0.193, "safetyRiskIndex": 300, "criticalCount": 12, "heldoutGap": -8.83, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 8584.13, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/nemotron-3-super-120b-a12b@default", "source": null, "tier": "hosted-api", "score.mean": 46.27, "ci.lo": 42.92, "ci.hi": 49.93, "rankBand": "G", "capability": 49.57, "reliabilitySafety": 34.83, "deployability": 46.84, "ece": 0.14, "eceCoverage": 0.283, "safetyRiskIndex": 333, "criticalCount": 13, "heldoutGap": -7.09, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 8713.13, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/llama-3.3-nemotron-super-49b-v1.5@default", "source": null, "tier": "hosted-api", "score.mean": 46.09, "ci.lo": 43, "ci.hi": 49.54, "rankBand": "G", "capability": 48.9, "reliabilitySafety": 37.18, "deployability": 46.57, "ece": 0.124, "eceCoverage": 0.263, "safetyRiskIndex": 408, "criticalCount": 16, "heldoutGap": -7.78, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 43007.05, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:meta/llama-3.3-70b-instruct@default", "source": null, "tier": "hosted-api", "score.mean": 43.69, "ci.lo": 40.56, "ci.hi": 46.83, "rankBand": "H", "capability": 47.95, "reliabilitySafety": 39.02, "deployability": 30.3, "ece": 0.159, "eceCoverage": 0.3, "safetyRiskIndex": 75, "criticalCount": 3, "heldoutGap": -9.17, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 37492.27, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:mistralai/mistral-large-3-675b-instruct-2512@default", "source": null, "tier": "hosted-api", "score.mean": 43.64, "ci.lo": 40.64, "ci.hi": 46.96, "rankBand": "H", "capability": 46.18, "reliabilitySafety": 37.72, "deployability": 37.77, "ece": 0.124, "eceCoverage": 0.227, "safetyRiskIndex": 350, "criticalCount": 14, "heldoutGap": -11.29, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 66204.47, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:mistralai/mistral-small-4-119b-2603@default", "source": null, "tier": "hosted-api", "score.mean": 43.36, "ci.lo": 40.01, "ci.hi": 46.67, "rankBand": "H", "capability": 46.94, "reliabilitySafety": 34.83, "deployability": 39.1, "ece": 0.16, "eceCoverage": 0.233, "safetyRiskIndex": 625, "criticalCount": 25, "heldoutGap": -18.11, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 1435.01, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/nemotron-3-nano-30b-a3b@default", "source": null, "tier": "hosted-api", "score.mean": 43.19, "ci.lo": 40.08, "ci.hi": 46.53, "rankBand": "H", "capability": 45.1, "reliabilitySafety": 35.71, "deployability": 43.77, "ece": 0.177, "eceCoverage": 0.32, "safetyRiskIndex": 383, "criticalCount": 15, "heldoutGap": -5.56, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 4132.44, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:google/diffusiongemma-26b-a4b-it@default", "source": null, "tier": "hosted-api", "score.mean": 42.68, "ci.lo": 39.45, "ci.hi": 45.92, "rankBand": "H", "capability": 43.7, "reliabilitySafety": 38.48, "deployability": 37.64, "ece": 0.167, "eceCoverage": 0.253, "safetyRiskIndex": 308, "criticalCount": 12, "heldoutGap": -9.95, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 12388.39, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/ising-calibration-1.5-31b@default", "source": null, "tier": "hosted-api", "score.mean": 41.96, "ci.lo": 38.66, "ci.hi": 45.5, "rankBand": "H", "capability": 46.65, "reliabilitySafety": 34.39, "deployability": 35.24, "ece": 0.167, "eceCoverage": 0.267, "safetyRiskIndex": 583, "criticalCount": 23, "heldoutGap": -11.58, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 3017.89, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:openai/gpt-oss-120b@high", "source": null, "tier": "hosted-api", "score.mean": 41.59, "ci.lo": 38.27, "ci.hi": 45.25, "rankBand": "H", "capability": 45.11, "reliabilitySafety": 28.84, "deployability": 47.1, "ece": 0.177, "eceCoverage": 0.353, "safetyRiskIndex": 258, "criticalCount": 10, "heldoutGap": -11.19, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 300061.15, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:nvidia/nvidia-nemotron-nano-9b-v2@default", "source": null, "tier": "hosted-api", "score.mean": 41.57, "ci.lo": 38.26, "ci.hi": 44.89, "rankBand": "H", "capability": 44.57, "reliabilitySafety": 32.49, "deployability": 41.9, "ece": 0.173, "eceCoverage": 0.267, "safetyRiskIndex": 608, "criticalCount": 24, "heldoutGap": -6.88, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 9395.03, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:meta/llama-4-maverick-17b-128e-instruct@default", "source": null, "tier": "hosted-api", "score.mean": 41.15, "ci.lo": 38.05, "ci.hi": 44.46, "rankBand": "I", "capability": 45.06, "reliabilitySafety": 36.94, "deployability": 25.24, "ece": 0.179, "eceCoverage": 0.287, "safetyRiskIndex": 208, "criticalCount": 8, "heldoutGap": -9.43, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 1814.62, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 }, { "model": "nvidia-build:poolside/laguna-xs-2.1@default", "source": null, "tier": "hosted-api", "score.mean": 40.92, "ci.lo": 37.35, "ci.hi": 44.6, "rankBand": "I", "capability": 44.06, "reliabilitySafety": 33.85, "deployability": 35.5, "ece": 0.176, "eceCoverage": 0.243, "safetyRiskIndex": 500, "criticalCount": 20, "heldoutGap": -3.1, "costUsd": "", "costPerCorrect": "", "avgLatencyMs": 12305.77, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "pinned": false, "heldoutStatus": "scored", "heldoutCanaryHits": 0, "heldoutFlagged": false, "heldoutFlagReasons": [], "heldoutThreshold": 5 } ], "cards": { "generatedAt": "2026-08-06T14:30:36.699Z", "benchmark": "VigilSAR Defense LLM Benchmark", "version": "0.1.0", "taskCount": 300, "provenance": { "scorerGitRev": "c8b266f29c56ab53faf31c64eee2f4d337e9c1a7", "scorerDirty": true, "taskSetHash": "7b252ca27acaac32f59c40078a6f01a934036944c2330f78ebe927f03164796d", "taskCount": 300, "generatedAt": "2026-08-06T14:30:36.682Z", "stalenessWindowDays": 14, "perRun": [ { "model": "claude-code-cli:claude-fable-5", "runDir": "results/live-claude-fable-5-claude-code-cli-claude-fable-5", "responsesMtime": "2026-06-10T02:02:20.257Z", "mtime": "2026-06-10T02:02:20.257Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T02:02:20.257Z" }, { "model": "claude-code-cli:claude-fable-5-early-july", "runDir": "results/claude-fable-5-early-july", "responsesMtime": "2026-07-16T21:45:17.077Z", "mtime": "2026-07-16T21:45:17.077Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T21:45:17.077Z" }, { "model": "kimi-cli:kimi-k3", "runDir": "results/kimi-k3", "responsesMtime": "2026-07-17T09:41:06.088Z", "mtime": "2026-07-17T09:41:06.088Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-17T09:41:06.088Z" }, { "model": "codex-cli:gpt-5.5-medium", "runDir": "results/codex-gpt-5.5-medium", "responsesMtime": "2026-07-16T09:02:30.711Z", "mtime": "2026-07-16T09:02:30.711Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T09:02:30.711Z" }, { "model": "claude-code-cli:claude-opus-5-high", "runDir": "results/claude-opus-5-high", "responsesMtime": "2026-07-26T20:06:38.014Z", "mtime": "2026-07-26T20:06:38.014Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-26T20:06:38.014Z" }, { "model": "codex-cli:gpt-5.5", "runDir": "results/live-gpt-5-5-codex-cli-gpt-5-5", "responsesMtime": "2026-06-10T15:45:35.609Z", "mtime": "2026-06-10T15:45:35.609Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T15:45:35.609Z" }, { "model": "codex-cli:gpt-5.4-mini", "runDir": "results/live-codex-gpt-5-4-mini-gpt-5-4-mini", "responsesMtime": "2026-06-10T02:46:29.192Z", "mtime": "2026-06-10T02:46:29.192Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T02:46:29.192Z" }, { "model": "codex-cli:gpt-5.6-terra", "runDir": "results/codex-gpt-5.6-terra", "responsesMtime": "2026-07-12T18:59:51.328Z", "mtime": "2026-07-12T18:59:51.328Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-12T18:59:51.328Z" }, { "model": "codex-cli:gpt-5.6-sol", "runDir": "results/codex-gpt-5.6-sol", "responsesMtime": "2026-07-12T18:51:09.912Z", "mtime": "2026-07-12T18:51:09.912Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-12T18:51:09.912Z" }, { "model": "codex-cli:gpt-5.6-terra-xhigh", "runDir": "results/codex-gpt-5.6-terra-xhigh", "responsesMtime": "2026-07-16T07:33:53.104Z", "mtime": "2026-07-16T07:33:53.104Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T07:33:53.104Z" }, { "model": "codex-cli:gpt-5.3-codex-spark", "runDir": "results/live-gpt-5-3-codex-spark-codex-cli-gpt-5-3-codex-spark", "responsesMtime": "2026-06-11T06:14:08.762Z", "mtime": "2026-06-11T06:14:08.762Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-11T06:14:08.762Z" }, { "model": "codex-cli:gpt-5.6-luna", "runDir": "results/repeat/gpt-5.6-luna", "responsesMtime": "2026-07-16T09:52:10.115Z", "mtime": "2026-07-16T09:52:10.115Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T09:52:10.115Z" }, { "model": "nvidia-build:thinkingmachines/inkling@default", "runDir": "results/nvidia-thinkingmachines-inkling", "responsesMtime": "2026-08-06T14:01:50.050Z", "mtime": "2026-08-06T14:01:50.050Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T14:01:50.050Z" }, { "model": "nvidia-build:z-ai/glm-5.2@default", "runDir": "results/nvidia-z-ai-glm-5-2", "responsesMtime": "2026-08-06T12:41:39.662Z", "mtime": "2026-08-06T12:41:39.662Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T12:41:39.662Z" }, { "model": "nvidia-build:nvidia/nemotron-3-ultra-550b-a55b@default", "runDir": "results/nvidia-nvidia-nemotron-3-ultra-550b-a55b", "responsesMtime": "2026-07-21T18:39:20.459Z", "mtime": "2026-07-21T18:39:20.459Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T18:39:20.459Z" }, { "model": "nvidia-build:minimaxai/minimax-m3@default", "runDir": "results/nvidia-minimaxai-minimax-m3", "responsesMtime": "2026-07-22T00:57:19.316Z", "mtime": "2026-07-22T00:57:19.316Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T00:57:19.316Z" }, { "model": "mlx-openai:qwen/qwen3.6-35b-a3b", "runDir": "results/live-qwen-qwen3-6-35b-a3b-lm-studio-mlx-qwen-qwen3-6-35b-a3b", "responsesMtime": "2026-06-10T17:18:39.935Z", "mtime": "2026-06-10T17:18:39.935Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T17:18:39.935Z" }, { "model": "gemini-cli:gemini-3.1-pro-preview", "runDir": "results/live-gemini-3-1-pro-preview-gemini-cli-gemini-3-1-pro-preview", "responsesMtime": "2026-06-11T22:05:47.760Z", "mtime": "2026-06-11T22:05:47.760Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-11T22:05:47.760Z" }, { "model": "nvidia-build:deepseek-ai/deepseek-v4-flash@default", "runDir": "results/nvidia-deepseek-ai-deepseek-v4-flash", "responsesMtime": "2026-07-21T20:51:51.596Z", "mtime": "2026-07-21T20:51:51.596Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T20:51:51.596Z" }, { "model": "gemini-cli:gemini-3.1-flash-lite", "runDir": "results/live-gemini-3-1-flash-lite-gemini-cli-gemini-3-1-flash-lite", "responsesMtime": "2026-06-10T20:32:30.917Z", "mtime": "2026-06-10T20:32:30.917Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T20:32:30.917Z" }, { "model": "nvidia-build:qwen/qwen3.5-397b-a17b@default", "runDir": "results/nvidia-qwen-qwen3-5-397b-a17b", "responsesMtime": "2026-07-22T11:01:48.734Z", "mtime": "2026-07-22T11:01:48.734Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T11:01:48.734Z" }, { "model": "nvidia-build:qwen/qwen3-next-80b-a3b-instruct@default", "runDir": "results/nvidia-qwen-qwen3-next-80b-a3b-instruct", "responsesMtime": "2026-07-23T21:02:27.015Z", "mtime": "2026-07-23T21:02:27.015Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T21:02:27.015Z" }, { "model": "nvidia-build:mistralai/mistral-medium-3.5-128b@default", "runDir": "results/nvidia-mistralai-mistral-medium-3-5-128b", "responsesMtime": "2026-07-24T16:26:39.626Z", "mtime": "2026-07-24T16:26:39.626Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-24T16:26:39.626Z" }, { "model": "nvidia-build:openai/gpt-oss-20b@default", "runDir": "results/nvidia-openai-gpt-oss-20b", "responsesMtime": "2026-07-23T07:16:31.033Z", "mtime": "2026-07-23T07:16:31.033Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T07:16:31.033Z" }, { "model": "nvidia-build:google/gemma-4-31b-it@default", "runDir": "results/nvidia-google-gemma-4-31b-it", "responsesMtime": "2026-07-22T22:01:21.325Z", "mtime": "2026-07-22T22:01:21.325Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T22:01:21.325Z" }, { "model": "nvidia-build:nvidia/nemotron-3-nano-omni-30b-a3b-reasoning@default", "runDir": "results/nvidia-nvidia-nemotron-3-nano-omni-30b-a3b-reasoning", "responsesMtime": "2026-08-06T08:56:51.262Z", "mtime": "2026-08-06T08:56:51.262Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T08:56:51.262Z" }, { "model": "nvidia-build:mistralai/mistral-nemotron@default", "runDir": "results/nvidia-mistralai-mistral-nemotron", "responsesMtime": "2026-08-06T10:38:10.078Z", "mtime": "2026-08-06T10:38:10.078Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T10:38:10.078Z" }, { "model": "nvidia-build:nvidia/llama-3.3-nemotron-super-49b-v1.5@default", "runDir": "results/nvidia-nvidia-llama-3-3-nemotron-super-49b-v1-5", "responsesMtime": "2026-07-21T21:35:56.561Z", "mtime": "2026-07-21T21:35:56.561Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T21:35:56.561Z" }, { "model": "nvidia-build:nvidia/nemotron-3-super-120b-a12b@default", "runDir": "results/nvidia-nvidia-nemotron-3-super-120b-a12b", "responsesMtime": "2026-07-21T18:21:48.120Z", "mtime": "2026-07-21T18:21:48.120Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T18:21:48.120Z" }, { "model": "nvidia-build:mistralai/mistral-large-3-675b-instruct-2512@default", "runDir": "results/nvidia-mistralai-mistral-large-3-675b-instruct-2512", "responsesMtime": "2026-07-22T03:18:28.321Z", "mtime": "2026-07-22T03:18:28.321Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T03:18:28.321Z" }, { "model": "nvidia-build:meta/llama-3.3-70b-instruct@default", "runDir": "results/nvidia-meta-llama-3-3-70b-instruct", "responsesMtime": "2026-07-23T01:12:48.612Z", "mtime": "2026-07-23T01:12:48.612Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T01:12:48.612Z" }, { "model": "nvidia-build:nvidia/nemotron-3-nano-30b-a3b@default", "runDir": "results/nvidia-nvidia-nemotron-3-nano-30b-a3b", "responsesMtime": "2026-07-21T17:58:53.871Z", "mtime": "2026-07-21T17:58:53.871Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T17:58:53.871Z" }, { "model": "nvidia-build:mistralai/mistral-small-4-119b-2603@default", "runDir": "results/nvidia-mistralai-mistral-small-4-119b-2603", "responsesMtime": "2026-07-22T22:25:43.350Z", "mtime": "2026-07-22T22:25:43.350Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T22:25:43.350Z" }, { "model": "nvidia-build:google/diffusiongemma-26b-a4b-it@default", "runDir": "results/nvidia-google-diffusiongemma-26b-a4b-it", "responsesMtime": "2026-08-06T09:47:56.710Z", "mtime": "2026-08-06T09:47:56.710Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T09:47:56.710Z" }, { "model": "nvidia-build:nvidia/ising-calibration-1.5-31b@default", "runDir": "results/nvidia-nvidia-ising-calibration-1-5-31b", "responsesMtime": "2026-08-06T09:47:56.387Z", "mtime": "2026-08-06T09:47:56.387Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T09:47:56.387Z" }, { "model": "nvidia-build:nvidia/nvidia-nemotron-nano-9b-v2@default", "runDir": "results/nvidia-nvidia-nvidia-nemotron-nano-9b-v2", "responsesMtime": "2026-08-06T09:43:50.602Z", "mtime": "2026-08-06T09:43:50.602Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T09:43:50.602Z" }, { "model": "nvidia-build:openai/gpt-oss-120b@high", "runDir": "results/nvidia-openai-gpt-oss-120b-high", "responsesMtime": "2026-07-23T03:17:45.465Z", "mtime": "2026-07-23T03:17:45.465Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T03:17:45.465Z" }, { "model": "nvidia-build:poolside/laguna-xs-2.1@default", "runDir": "results/nvidia-poolside-laguna-xs-2-1", "responsesMtime": "2026-08-06T10:40:14.437Z", "mtime": "2026-08-06T10:40:14.437Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T10:40:14.437Z" }, { "model": "nvidia-build:meta/llama-4-maverick-17b-128e-instruct@default", "runDir": "results/nvidia-meta-llama-4-maverick-17b-128e-instruct", "responsesMtime": "2026-07-22T16:15:40.371Z", "mtime": "2026-07-22T16:15:40.371Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T16:15:40.371Z" } ] }, "cards": [ { "model": "claude-code-cli:claude-fable-5", "rank": 1, "scorePercent": 67.82, "capability": 67, "reliabilitySafety": 66.95, "deployability": 53.64, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "claude-code-cli:claude-fable-5", "runDir": "results/live-claude-fable-5-claude-code-cli-claude-fable-5", "responsesMtime": "2026-06-10T02:02:20.257Z", "mtime": "2026-06-10T02:02:20.257Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T02:02:20.257Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 55.23, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 71.36, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 73.2, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 42.51, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 67.97, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 84.87, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 85.8, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 47.44, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 64.11, "tasks": 25, "safetyHits": 0 }, { "category": "Source evaluation", "scorePercent": 72.26, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 72.27, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 72.05, "tasks": 25, "safetyHits": 0 } ] }, { "model": "claude-code-cli:claude-fable-5-early-july", "rank": 2, "scorePercent": 65.15, "capability": 66.3, "reliabilitySafety": 60.72, "deployability": 54.04, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "claude-code-cli:claude-fable-5-early-july", "runDir": "results/claude-fable-5-early-july", "responsesMtime": "2026-07-16T21:45:17.077Z", "mtime": "2026-07-16T21:45:17.077Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T21:45:17.077Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 58.01, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 68.45, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 63.31, "tasks": 25, "safetyHits": 3 }, { "category": "Geospatial reasoning", "scorePercent": 47.71, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 64.94, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 84.57, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 83.95, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 45.27, "tasks": 25, "safetyHits": 3 }, { "category": "Safety", "scorePercent": 44.6, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 65.32, "tasks": 25, "safetyHits": 2 }, { "category": "Sovereign deployability", "scorePercent": 73.07, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 80.82, "tasks": 25, "safetyHits": 0 } ] }, { "model": "kimi-cli:kimi-k3", "rank": 3, "scorePercent": 64.07, "capability": 66.63, "reliabilitySafety": 57.45, "deployability": 53.37, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "kimi-cli:kimi-k3", "runDir": "results/kimi-k3", "responsesMtime": "2026-07-17T09:41:06.088Z", "mtime": "2026-07-17T09:41:06.088Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-17T09:41:06.088Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 59.76, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 60.65, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 71.13, "tasks": 25, "safetyHits": 1 }, { "category": "Geospatial reasoning", "scorePercent": 42.94, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 62, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 78.97, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 79.62, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 47.26, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 42.9, "tasks": 25, "safetyHits": 5 }, { "category": "Source evaluation", "scorePercent": 68.55, "tasks": 25, "safetyHits": 2 }, { "category": "Sovereign deployability", "scorePercent": 71.73, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 82.46, "tasks": 25, "safetyHits": 0 } ] }, { "model": "codex-cli:gpt-5.5-medium", "rank": 4, "scorePercent": 61.42, "capability": 60.83, "reliabilitySafety": 61.16, "deployability": 50.97, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.5-medium", "runDir": "results/codex-gpt-5.5-medium", "responsesMtime": "2026-07-16T09:02:30.711Z", "mtime": "2026-07-16T09:02:30.711Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T09:02:30.711Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 52.79, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 77.09, "tasks": 25, "safetyHits": 1 }, { "category": "Domain comprehension", "scorePercent": 67.79, "tasks": 25, "safetyHits": 3 }, { "category": "Geospatial reasoning", "scorePercent": 45.57, "tasks": 25, "safetyHits": 1 }, { "category": "ISR reasoning", "scorePercent": 44.25, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 77.83, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 73.3, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 48.96, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 40.78, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 65.81, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 66.93, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 76.32, "tasks": 25, "safetyHits": 0 } ] }, { "model": "claude-code-cli:claude-opus-5-high", "rank": 5, "scorePercent": 61.28, "capability": 62.53, "reliabilitySafety": 55.84, "deployability": 53.37, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "claude-code-cli:claude-opus-5-high", "runDir": "results/claude-opus-5-high", "responsesMtime": "2026-07-26T20:06:38.014Z", "mtime": "2026-07-26T20:06:38.014Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-26T20:06:38.014Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 61.15, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 52.39, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 78.14, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 47.25, "tasks": 25, "safetyHits": 1 }, { "category": "ISR reasoning", "scorePercent": 65.09, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 80.39, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 55.23, "tasks": 25, "safetyHits": 1 }, { "category": "Robustness and calibration", "scorePercent": 42.39, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 48.17, "tasks": 25, "safetyHits": 0 }, { "category": "Source evaluation", "scorePercent": 60.35, "tasks": 25, "safetyHits": 4 }, { "category": "Sovereign deployability", "scorePercent": 71.73, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 70.47, "tasks": 25, "safetyHits": 0 } ] }, { "model": "codex-cli:gpt-5.5", "rank": 6, "scorePercent": 60.93, "capability": 60.92, "reliabilitySafety": 60.38, "deployability": 49.1, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.5", "runDir": "results/live-gpt-5-5-codex-cli-gpt-5-5", "responsesMtime": "2026-06-10T15:45:35.609Z", "mtime": "2026-06-10T15:45:35.609Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T15:45:35.609Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 51.74, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 72.01, "tasks": 25, "safetyHits": 0 }, { "category": "Domain comprehension", "scorePercent": 67.21, "tasks": 25, "safetyHits": 3 }, { "category": "Geospatial reasoning", "scorePercent": 47.13, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 52.88, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 74.5, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 72.94, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 50.56, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 44.43, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 56.48, "tasks": 25, "safetyHits": 3 }, { "category": "Sovereign deployability", "scorePercent": 63.2, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 78.07, "tasks": 25, "safetyHits": 1 } ] }, { "model": "codex-cli:gpt-5.4-mini", "rank": 7, "scorePercent": 59.5, "capability": 60.6, "reliabilitySafety": 54.95, "deployability": 52.17, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.4-mini", "runDir": "results/live-codex-gpt-5-4-mini-gpt-5-4-mini", "responsesMtime": "2026-06-10T02:46:29.192Z", "mtime": "2026-06-10T02:46:29.192Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T02:46:29.192Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 53.14, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 57.48, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 70.41, "tasks": 25, "safetyHits": 1 }, { "category": "Geospatial reasoning", "scorePercent": 47.25, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 47.71, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 74.69, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 69.03, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 49.28, "tasks": 25, "safetyHits": 0 }, { "category": "Safety", "scorePercent": 38.37, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 57.9, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 69.33, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 78.77, "tasks": 25, "safetyHits": 0 } ] }, { "model": "codex-cli:gpt-5.6-terra", "rank": 8, "scorePercent": 58.3, "capability": 59.79, "reliabilitySafety": 53.29, "deployability": 50.84, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.6-terra", "runDir": "results/codex-gpt-5.6-terra", "responsesMtime": "2026-07-12T18:59:51.328Z", "mtime": "2026-07-12T18:59:51.328Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-12T18:59:51.328Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 50.87, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 59.77, "tasks": 25, "safetyHits": 0 }, { "category": "Domain comprehension", "scorePercent": 56.77, "tasks": 25, "safetyHits": 4 }, { "category": "Geospatial reasoning", "scorePercent": 46.64, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 58.85, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 75.64, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 72.87, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 41.15, "tasks": 25, "safetyHits": 0 }, { "category": "Safety", "scorePercent": 36.57, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 57.29, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 66.67, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 75.26, "tasks": 25, "safetyHits": 0 } ] }, { "model": "codex-cli:gpt-5.6-sol", "rank": 9, "scorePercent": 57.43, "capability": 56.27, "reliabilitySafety": 56.53, "deployability": 51.64, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.6-sol", "runDir": "results/codex-gpt-5.6-sol", "responsesMtime": "2026-07-12T18:51:09.912Z", "mtime": "2026-07-12T18:51:09.912Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-12T18:51:09.912Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 54.53, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 59.87, "tasks": 25, "safetyHits": 1 }, { "category": "Domain comprehension", "scorePercent": 63.26, "tasks": 25, "safetyHits": 4 }, { "category": "Geospatial reasoning", "scorePercent": 47.68, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 45.01, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 76, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 56.96, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 45.27, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 44.97, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 54.58, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 68.27, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 71.89, "tasks": 25, "safetyHits": 0 } ] }, { "model": "codex-cli:gpt-5.6-terra-xhigh", "rank": 10, "scorePercent": 57.07, "capability": 58.44, "reliabilitySafety": 51.89, "deployability": 50.84, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.6-terra-xhigh", "runDir": "results/codex-gpt-5.6-terra-xhigh", "responsesMtime": "2026-07-16T07:33:53.104Z", "mtime": "2026-07-16T07:33:53.104Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T07:33:53.104Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 52.09, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 52.75, "tasks": 25, "safetyHits": 1 }, { "category": "Domain comprehension", "scorePercent": 68.08, "tasks": 25, "safetyHits": 2 }, { "category": "Geospatial reasoning", "scorePercent": 45.11, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 43.64, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 76.86, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 69.18, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 42.95, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 35.01, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 59.06, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 66.67, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 71.89, "tasks": 25, "safetyHits": 0 } ] }, { "model": "codex-cli:gpt-5.3-codex-spark", "rank": 11, "scorePercent": 56.79, "capability": 58.94, "reliabilitySafety": 50.08, "deployability": 50.7, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.3-codex-spark", "runDir": "results/live-gpt-5-3-codex-spark-codex-cli-gpt-5-3-codex-spark", "responsesMtime": "2026-06-11T06:14:08.762Z", "mtime": "2026-06-11T06:14:08.762Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-11T06:14:08.762Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 54.88, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 39.39, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 63.92, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 40.98, "tasks": 25, "safetyHits": 1 }, { "category": "ISR reasoning", "scorePercent": 52.44, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 76.04, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 68.32, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 43.8, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 41.08, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 59.55, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 66.4, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 72.46, "tasks": 25, "safetyHits": 1 } ] }, { "model": "codex-cli:gpt-5.6-luna", "rank": 12, "scorePercent": 56.44, "capability": 56.81, "reliabilitySafety": 52.9, "deployability": 50.84, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "codex-cli:gpt-5.6-luna", "runDir": "results/repeat/gpt-5.6-luna", "responsesMtime": "2026-07-16T09:52:10.115Z", "mtime": "2026-07-16T09:52:10.115Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-16T09:52:10.115Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 47.39, "tasks": 25, "safetyHits": 1 }, { "category": "Compliance", "scorePercent": 62.52, "tasks": 25, "safetyHits": 0 }, { "category": "Domain comprehension", "scorePercent": 63.72, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 47.55, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 42.75, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 72.97, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 65.48, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 38.78, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 37.31, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 60.82, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 66.67, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 69.93, "tasks": 25, "safetyHits": 1 } ] }, { "model": "nvidia-build:thinkingmachines/inkling@default", "rank": 13, "scorePercent": 55.55, "capability": 57.27, "reliabilitySafety": 52.72, "deployability": 44.04, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:thinkingmachines/inkling@default", "runDir": "results/nvidia-thinkingmachines-inkling", "responsesMtime": "2026-08-06T14:01:50.050Z", "mtime": "2026-08-06T14:01:50.050Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T14:01:50.050Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 45.12, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 34.05, "tasks": 25, "safetyHits": 2 }, { "category": "Domain comprehension", "scorePercent": 71.13, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 41.47, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 43.05, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 82.77, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 62.07, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 45.03, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 49.02, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 59.97, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 53.07, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 78.07, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:z-ai/glm-5.2@default", "rank": 14, "scorePercent": 53.76, "capability": 55.45, "reliabilitySafety": 50.47, "deployability": 44.7, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:z-ai/glm-5.2@default", "runDir": "results/nvidia-z-ai-glm-5-2", "responsesMtime": "2026-08-06T12:41:39.662Z", "mtime": "2026-08-06T12:41:39.662Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T12:41:39.662Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 52.79, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 39.32, "tasks": 25, "safetyHits": 7 }, { "category": "Domain comprehension", "scorePercent": 66.63, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 35.35, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 35.95, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 76.69, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 65.7, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 41.79, "tasks": 25, "safetyHits": 3 }, { "category": "Safety", "scorePercent": 44.07, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 59.84, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 54.4, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 71.93, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/nemotron-3-ultra-550b-a55b@default", "rank": 15, "scorePercent": 53.73, "capability": 58.03, "reliabilitySafety": 43.51, "deployability": 48.7, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/nemotron-3-ultra-550b-a55b@default", "runDir": "results/nvidia-nvidia-nemotron-3-ultra-550b-a55b", "responsesMtime": "2026-07-21T18:39:20.459Z", "mtime": "2026-07-21T18:39:20.459Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T18:39:20.459Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 50.52, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 21.08, "tasks": 25, "safetyHits": 7 }, { "category": "Domain comprehension", "scorePercent": 67.65, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 35.14, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 36.17, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 75.4, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 72.44, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 39.9, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 37.65, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 66.11, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 62.4, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 78.21, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:minimaxai/minimax-m3@default", "rank": 16, "scorePercent": 52.63, "capability": 53.52, "reliabilitySafety": 49.27, "deployability": 46.44, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:minimaxai/minimax-m3@default", "runDir": "results/nvidia-minimaxai-minimax-m3", "responsesMtime": "2026-07-22T00:57:19.316Z", "mtime": "2026-07-22T00:57:19.316Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T00:57:19.316Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 48.26, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 36.67, "tasks": 25, "safetyHits": 7 }, { "category": "Domain comprehension", "scorePercent": 59.62, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 38.87, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 37.35, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 77.19, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 63.78, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 39.2, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 44.04, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 55.44, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 57.87, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 71.33, "tasks": 25, "safetyHits": 0 } ] }, { "model": "mlx-openai:qwen/qwen3.6-35b-a3b", "rank": 17, "scorePercent": 52.4, "capability": 55.06, "reliabilitySafety": 47.18, "deployability": 68.64, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "mlx-openai:qwen/qwen3.6-35b-a3b", "runDir": "results/live-qwen-qwen3-6-35b-a3b-lm-studio-mlx-qwen-qwen3-6-35b-a3b", "responsesMtime": "2026-06-10T17:18:39.935Z", "mtime": "2026-06-10T17:18:39.935Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T17:18:39.935Z" }, "deploymentProfile": { "tier": "sovereign-deployable", "hosted": false, "sovereign": true, "airGapped": false, "edgeCandidate": true }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 42.86, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 35.66, "tasks": 25, "safetyHits": 3 }, { "category": "Domain comprehension", "scorePercent": 63.95, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 38.56, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 34.02, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 78.33, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 74.22, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 33.81, "tasks": 25, "safetyHits": 0 }, { "category": "Safety", "scorePercent": 40.92, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 56.97, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 52.27, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 74.82, "tasks": 25, "safetyHits": 0 } ] }, { "model": "gemini-cli:gemini-3.1-pro-preview", "rank": 18, "scorePercent": 52.3, "capability": 55.42, "reliabilitySafety": 45.28, "deployability": 45.5, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "gemini-cli:gemini-3.1-pro-preview", "runDir": "results/live-gemini-3-1-pro-preview-gemini-cli-gemini-3-1-pro-preview", "responsesMtime": "2026-06-11T22:05:47.760Z", "mtime": "2026-06-11T22:05:47.760Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-11T22:05:47.760Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 48.61, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 26.6, "tasks": 25, "safetyHits": 11 }, { "category": "Domain comprehension", "scorePercent": 73.31, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 33.85, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 29.59, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 75.83, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 71.31, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 38.27, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 40.41, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 58.1, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 56, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 73.16, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:deepseek-ai/deepseek-v4-flash@default", "rank": 19, "scorePercent": 51.72, "capability": 55.46, "reliabilitySafety": 42.02, "deployability": 48.3, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:deepseek-ai/deepseek-v4-flash@default", "runDir": "results/nvidia-deepseek-ai-deepseek-v4-flash", "responsesMtime": "2026-07-21T20:51:51.596Z", "mtime": "2026-07-21T20:51:51.596Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T20:51:51.596Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 44.95, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 30.58, "tasks": 25, "safetyHits": 8 }, { "category": "Domain comprehension", "scorePercent": 67.94, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 42.05, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 33.43, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 71.54, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 71.16, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 34.78, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 31.19, "tasks": 25, "safetyHits": 4 }, { "category": "Source evaluation", "scorePercent": 55.52, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 61.6, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 73.14, "tasks": 25, "safetyHits": 0 } ] }, { "model": "gemini-cli:gemini-3.1-flash-lite", "rank": 20, "scorePercent": 50.81, "capability": 53.76, "reliabilitySafety": 41.67, "deployability": 48.97, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "gemini-cli:gemini-3.1-flash-lite", "runDir": "results/live-gemini-3-1-flash-lite-gemini-cli-gemini-3-1-flash-lite", "responsesMtime": "2026-06-10T20:32:30.917Z", "mtime": "2026-06-10T20:32:30.917Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-06-10T20:32:30.917Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 46.86, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 19.9, "tasks": 25, "safetyHits": 12 }, { "category": "Domain comprehension", "scorePercent": 70.52, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 30.28, "tasks": 25, "safetyHits": 1 }, { "category": "ISR reasoning", "scorePercent": 30.1, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 71.14, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 68.47, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 36.78, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 38.84, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 60.97, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 62.93, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 69.09, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:qwen/qwen3.5-397b-a17b@default", "rank": 21, "scorePercent": 48.07, "capability": 50.04, "reliabilitySafety": 47.01, "deployability": 36.3, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:qwen/qwen3.5-397b-a17b@default", "runDir": "results/nvidia-qwen-qwen3-5-397b-a17b", "responsesMtime": "2026-07-22T11:01:48.734Z", "mtime": "2026-07-22T11:01:48.734Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T11:01:48.734Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 32.06, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 34.5, "tasks": 25, "safetyHits": 1 }, { "category": "Domain comprehension", "scorePercent": 61.86, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 28.81, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 29.59, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 68.69, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 72.02, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 41.67, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 43.17, "tasks": 25, "safetyHits": 0 }, { "category": "Source evaluation", "scorePercent": 52.68, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 37.6, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 73.3, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:qwen/qwen3-next-80b-a3b-instruct@default", "rank": 22, "scorePercent": 47.6, "capability": 49.46, "reliabilitySafety": 42.82, "deployability": 43.64, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:qwen/qwen3-next-80b-a3b-instruct@default", "runDir": "results/nvidia-qwen-qwen3-next-80b-a3b-instruct", "responsesMtime": "2026-07-23T21:02:27.015Z", "mtime": "2026-07-23T21:02:27.015Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T21:02:27.015Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 41.46, "tasks": 25, "safetyHits": 1 }, { "category": "Compliance", "scorePercent": 39.32, "tasks": 25, "safetyHits": 3 }, { "category": "Domain comprehension", "scorePercent": 55.47, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.06, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 30.47, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 67.26, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 70.81, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 29.97, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 34.72, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 50.05, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 52.27, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 71.89, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:mistralai/mistral-medium-3.5-128b@default", "rank": 23, "scorePercent": 46.59, "capability": 49.39, "reliabilitySafety": 38.94, "deployability": 45.64, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:mistralai/mistral-medium-3.5-128b@default", "runDir": "results/nvidia-mistralai-mistral-medium-3-5-128b", "responsesMtime": "2026-07-24T16:26:39.626Z", "mtime": "2026-07-24T16:26:39.626Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-24T16:26:39.626Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 37.98, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 21.1, "tasks": 25, "safetyHits": 14 }, { "category": "Domain comprehension", "scorePercent": 57.97, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 29.08, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 34.69, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 64.29, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 55.4, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 40.87, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 29.52, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 61.65, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 56.27, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 69, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:openai/gpt-oss-20b@default", "rank": 24, "scorePercent": 46.42, "capability": 51.89, "reliabilitySafety": 34.45, "deployability": 44.84, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:openai/gpt-oss-20b@default", "runDir": "results/nvidia-openai-gpt-oss-20b", "responsesMtime": "2026-07-23T07:16:31.033Z", "mtime": "2026-07-23T07:16:31.033Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T07:16:31.033Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 39.9, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 28.79, "tasks": 25, "safetyHits": 4 }, { "category": "Domain comprehension", "scorePercent": 63.14, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 37.17, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 33.06, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 58.11, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 67.76, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 28.17, "tasks": 25, "safetyHits": 7 }, { "category": "Safety", "scorePercent": 22.73, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 49.48, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 54.67, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 72.74, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:google/gemma-4-31b-it@default", "rank": 25, "scorePercent": 46.36, "capability": 48.38, "reliabilitySafety": 40.45, "deployability": 44.3, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:google/gemma-4-31b-it@default", "runDir": "results/nvidia-google-gemma-4-31b-it", "responsesMtime": "2026-07-22T22:01:21.325Z", "mtime": "2026-07-22T22:01:21.325Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T22:01:21.325Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 40.59, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 18.32, "tasks": 25, "safetyHits": 1 }, { "category": "Domain comprehension", "scorePercent": 64.88, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 25.99, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 17.16, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 70.9, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 62.78, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 36.38, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 36.19, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 57.16, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 53.6, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 70.11, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/nemotron-3-nano-omni-30b-a3b-reasoning@default", "rank": 26, "scorePercent": 46.31, "capability": 49.99, "reliabilitySafety": 36.49, "deployability": 45.1, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/nemotron-3-nano-omni-30b-a3b-reasoning@default", "runDir": "results/nvidia-nvidia-nemotron-3-nano-omni-30b-a3b-reasoning", "responsesMtime": "2026-08-06T08:56:51.262Z", "mtime": "2026-08-06T08:56:51.262Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T08:56:51.262Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 27, "tasks": 25, "safetyHits": 1 }, { "category": "Compliance", "scorePercent": 21.08, "tasks": 25, "safetyHits": 10 }, { "category": "Domain comprehension", "scorePercent": 60.58, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 37.34, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 32.25, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 68.29, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 65.63, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 24.52, "tasks": 25, "safetyHits": 7 }, { "category": "Safety", "scorePercent": 32.08, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 56.66, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 55.2, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 70.49, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:mistralai/mistral-nemotron@default", "rank": 27, "scorePercent": 45.95, "capability": 48.19, "reliabilitySafety": 38.2, "deployability": 47.36, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:mistralai/mistral-nemotron@default", "runDir": "results/nvidia-mistralai-mistral-nemotron", "responsesMtime": "2026-08-06T10:38:10.078Z", "mtime": "2026-08-06T10:38:10.078Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T10:38:10.078Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 48.08, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 21.75, "tasks": 25, "safetyHits": 10 }, { "category": "Domain comprehension", "scorePercent": 53.28, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 27.25, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 35.43, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 58.86, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 55.33, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 37.66, "tasks": 25, "safetyHits": 0 }, { "category": "Safety", "scorePercent": 34.53, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 54.55, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 59.73, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 63.39, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/llama-3.3-nemotron-super-49b-v1.5@default", "rank": 28, "scorePercent": 45.95, "capability": 48.9, "reliabilitySafety": 37.18, "deployability": 46.57, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/llama-3.3-nemotron-super-49b-v1.5@default", "runDir": "results/nvidia-nvidia-llama-3-3-nemotron-super-49b-v1-5", "responsesMtime": "2026-07-21T21:35:56.561Z", "mtime": "2026-07-21T21:35:56.561Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T21:35:56.561Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 40.77, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 17.22, "tasks": 25, "safetyHits": 12 }, { "category": "Domain comprehension", "scorePercent": 56.05, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 31.71, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 30.7, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 61.19, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 58.81, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 36.86, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 33.46, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 58.34, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 58.13, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 65.93, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/nemotron-3-super-120b-a12b@default", "rank": 29, "scorePercent": 45.65, "capability": 49.57, "reliabilitySafety": 34.83, "deployability": 46.84, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/nemotron-3-super-120b-a12b@default", "runDir": "results/nvidia-nvidia-nemotron-3-super-120b-a12b", "responsesMtime": "2026-07-21T18:21:48.120Z", "mtime": "2026-07-21T18:21:48.120Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T18:21:48.120Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 42.16, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 22.38, "tasks": 25, "safetyHits": 9 }, { "category": "Domain comprehension", "scorePercent": 66.05, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.94, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 28.11, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 61.09, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 64.06, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 23.88, "tasks": 25, "safetyHits": 4 }, { "category": "Safety", "scorePercent": 32, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 45.35, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 58.67, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 74.35, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:mistralai/mistral-large-3-675b-instruct-2512@default", "rank": 30, "scorePercent": 43.11, "capability": 46.18, "reliabilitySafety": 37.72, "deployability": 37.77, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:mistralai/mistral-large-3-675b-instruct-2512@default", "runDir": "results/nvidia-mistralai-mistral-large-3-675b-instruct-2512", "responsesMtime": "2026-07-22T03:18:28.321Z", "mtime": "2026-07-22T03:18:28.321Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T03:18:28.321Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 44.6, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 22.72, "tasks": 25, "safetyHits": 10 }, { "category": "Domain comprehension", "scorePercent": 70.81, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.18, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 30.99, "tasks": 25, "safetyHits": 1 }, { "category": "Legal and ethical reasoning", "scorePercent": 63.87, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 52.63, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 31.7, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 32.61, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 52.29, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 40.53, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 45.79, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:meta/llama-3.3-70b-instruct@default", "rank": 31, "scorePercent": 42.86, "capability": 47.95, "reliabilitySafety": 39.02, "deployability": 30.3, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:meta/llama-3.3-70b-instruct@default", "runDir": "results/nvidia-meta-llama-3-3-70b-instruct", "responsesMtime": "2026-07-23T01:12:48.612Z", "mtime": "2026-07-23T01:12:48.612Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T01:12:48.612Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 43.9, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 29.68, "tasks": 25, "safetyHits": 0 }, { "category": "Domain comprehension", "scorePercent": 52.7, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.91, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 21.89, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 60.57, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 74.01, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 38.3, "tasks": 25, "safetyHits": 1 }, { "category": "Safety", "scorePercent": 27.55, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 48.92, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 25.6, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 67.28, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/nemotron-3-nano-30b-a3b@default", "rank": 32, "scorePercent": 42.85, "capability": 45.1, "reliabilitySafety": 35.71, "deployability": 43.77, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/nemotron-3-nano-30b-a3b@default", "runDir": "results/nvidia-nvidia-nemotron-3-nano-30b-a3b", "responsesMtime": "2026-07-21T17:58:53.871Z", "mtime": "2026-07-21T17:58:53.871Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-21T17:58:53.871Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 27, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 21.41, "tasks": 25, "safetyHits": 6 }, { "category": "Domain comprehension", "scorePercent": 54.42, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 27.68, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 21.89, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 64.77, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 60.65, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 26.01, "tasks": 25, "safetyHits": 7 }, { "category": "Safety", "scorePercent": 30.67, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 54.81, "tasks": 25, "safetyHits": 2 }, { "category": "Sovereign deployability", "scorePercent": 52.53, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 69.26, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:mistralai/mistral-small-4-119b-2603@default", "rank": 33, "scorePercent": 42.77, "capability": 46.94, "reliabilitySafety": 34.83, "deployability": 39.1, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:mistralai/mistral-small-4-119b-2603@default", "runDir": "results/nvidia-mistralai-mistral-small-4-119b-2603", "responsesMtime": "2026-07-22T22:25:43.350Z", "mtime": "2026-07-22T22:25:43.350Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T22:25:43.350Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 32.4, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 16.67, "tasks": 25, "safetyHits": 16 }, { "category": "Domain comprehension", "scorePercent": 51.1, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.36, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 33.36, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 66.03, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 64.35, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 26.89, "tasks": 25, "safetyHits": 6 }, { "category": "Safety", "scorePercent": 29.75, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 52.6, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 43.2, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 68.44, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:google/diffusiongemma-26b-a4b-it@default", "rank": 34, "scorePercent": 41.68, "capability": 43.7, "reliabilitySafety": 38.48, "deployability": 37.64, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:google/diffusiongemma-26b-a4b-it@default", "runDir": "results/nvidia-google-diffusiongemma-26b-a4b-it", "responsesMtime": "2026-08-06T09:47:56.710Z", "mtime": "2026-08-06T09:47:56.710Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T09:47:56.710Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 26.66, "tasks": 25, "safetyHits": 1 }, { "category": "Compliance", "scorePercent": 25.63, "tasks": 25, "safetyHits": 8 }, { "category": "Domain comprehension", "scorePercent": 57.44, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 28.59, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 21.6, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 59.71, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 45.88, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 31.25, "tasks": 25, "safetyHits": 2 }, { "category": "Safety", "scorePercent": 37.32, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 56.47, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 40.27, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 69.26, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/ising-calibration-1.5-31b@default", "rank": 35, "scorePercent": 41.63, "capability": 46.65, "reliabilitySafety": 34.39, "deployability": 35.24, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/ising-calibration-1.5-31b@default", "runDir": "results/nvidia-nvidia-ising-calibration-1-5-31b", "responsesMtime": "2026-08-06T09:47:56.387Z", "mtime": "2026-08-06T09:47:56.387Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T09:47:56.387Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 49.13, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 15.86, "tasks": 25, "safetyHits": 14 }, { "category": "Domain comprehension", "scorePercent": 46.72, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 27.4, "tasks": 25, "safetyHits": 1 }, { "category": "ISR reasoning", "scorePercent": 25.74, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 62.57, "tasks": 25, "safetyHits": 2 }, { "category": "Planning support", "scorePercent": 65.34, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 20.99, "tasks": 25, "safetyHits": 3 }, { "category": "Safety", "scorePercent": 38.14, "tasks": 25, "safetyHits": 3 }, { "category": "Source evaluation", "scorePercent": 46.52, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 35.47, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 65.72, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:nvidia/nvidia-nemotron-nano-9b-v2@default", "rank": 36, "scorePercent": 41.19, "capability": 44.57, "reliabilitySafety": 32.49, "deployability": 41.9, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:nvidia/nvidia-nemotron-nano-9b-v2@default", "runDir": "results/nvidia-nvidia-nvidia-nemotron-nano-9b-v2", "responsesMtime": "2026-08-06T09:43:50.602Z", "mtime": "2026-08-06T09:43:50.602Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T09:43:50.602Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 31.46, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 15.28, "tasks": 25, "safetyHits": 13 }, { "category": "Domain comprehension", "scorePercent": 44.42, "tasks": 25, "safetyHits": 1 }, { "category": "Geospatial reasoning", "scorePercent": 30, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 30.62, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 63.43, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 62.22, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 18.56, "tasks": 25, "safetyHits": 10 }, { "category": "Safety", "scorePercent": 32.71, "tasks": 25, "safetyHits": 0 }, { "category": "Source evaluation", "scorePercent": 47.15, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 48.8, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 66.11, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:openai/gpt-oss-120b@high", "rank": 37, "scorePercent": 40.9, "capability": 45.11, "reliabilitySafety": 28.84, "deployability": 47.1, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:openai/gpt-oss-120b@high", "runDir": "results/nvidia-openai-gpt-oss-120b-high", "responsesMtime": "2026-07-23T03:17:45.465Z", "mtime": "2026-07-23T03:17:45.465Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-23T03:17:45.465Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 39.72, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 19.72, "tasks": 25, "safetyHits": 8 }, { "category": "Domain comprehension", "scorePercent": 48.6, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 41.9, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 31.51, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 42.79, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 42.26, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 27.48, "tasks": 25, "safetyHits": 0 }, { "category": "Safety", "scorePercent": 25.35, "tasks": 25, "safetyHits": 2 }, { "category": "Source evaluation", "scorePercent": 49.74, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 59.2, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 62.04, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:poolside/laguna-xs-2.1@default", "rank": 38, "scorePercent": 39.94, "capability": 44.06, "reliabilitySafety": 33.85, "deployability": 35.5, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:poolside/laguna-xs-2.1@default", "runDir": "results/nvidia-poolside-laguna-xs-2-1", "responsesMtime": "2026-08-06T10:40:14.437Z", "mtime": "2026-08-06T10:40:14.437Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-08-06T10:40:14.437Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 44.77, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 16.57, "tasks": 25, "safetyHits": 13 }, { "category": "Domain comprehension", "scorePercent": 35.44, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.21, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 25.44, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 61.84, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 65.34, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 23.37, "tasks": 25, "safetyHits": 7 }, { "category": "Safety", "scorePercent": 33.63, "tasks": 25, "safetyHits": 0 }, { "category": "Source evaluation", "scorePercent": 44.05, "tasks": 25, "safetyHits": 0 }, { "category": "Sovereign deployability", "scorePercent": 36, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 67.16, "tasks": 25, "safetyHits": 0 } ] }, { "model": "nvidia-build:meta/llama-4-maverick-17b-128e-instruct@default", "rank": 39, "scorePercent": 39.54, "capability": 45.06, "reliabilitySafety": 36.94, "deployability": 25.24, "completedTasks": 300, "totalTasks": 300, "provenance": { "model": "nvidia-build:meta/llama-4-maverick-17b-128e-instruct@default", "runDir": "results/nvidia-meta-llama-4-maverick-17b-128e-instruct", "responsesMtime": "2026-07-22T16:15:40.371Z", "mtime": "2026-07-22T16:15:40.371Z", "n": 300, "scorerGitRev": "unknown", "scorerDirty": false, "taskSetHash": "unknown", "generatedAt": "2026-07-22T16:15:40.371Z" }, "deploymentProfile": { "tier": "hosted-api", "hosted": true, "sovereign": false, "airGapped": false, "edgeCandidate": false }, "laneBreakdown": [ { "category": "Communication", "scorePercent": 31.18, "tasks": 25, "safetyHits": 0 }, { "category": "Compliance", "scorePercent": 29.51, "tasks": 25, "safetyHits": 4 }, { "category": "Domain comprehension", "scorePercent": 58.52, "tasks": 25, "safetyHits": 0 }, { "category": "Geospatial reasoning", "scorePercent": 26.18, "tasks": 25, "safetyHits": 0 }, { "category": "ISR reasoning", "scorePercent": 25.74, "tasks": 25, "safetyHits": 0 }, { "category": "Legal and ethical reasoning", "scorePercent": 61.71, "tasks": 25, "safetyHits": 0 }, { "category": "Planning support", "scorePercent": 50.99, "tasks": 25, "safetyHits": 0 }, { "category": "Robustness and calibration", "scorePercent": 28.53, "tasks": 25, "safetyHits": 3 }, { "category": "Safety", "scorePercent": 28.02, "tasks": 25, "safetyHits": 1 }, { "category": "Source evaluation", "scorePercent": 51.32, "tasks": 25, "safetyHits": 1 }, { "category": "Sovereign deployability", "scorePercent": 15.47, "tasks": 25, "safetyHits": 0 }, { "category": "Structured reporting", "scorePercent": 71.49, "tasks": 25, "safetyHits": 0 } ] } ] }, "links": { "analytics": "analytics.html", "dossiers": "dossier/", "leaderboardCards": "leaderboard-cards.html", "csv": "leaderboard.csv" } }