{ "status": "preliminary synthetic router experiment", "source_revision": "da3479f", "corpus_sha256": "2b277ecc9b1b5c8d91072a8e51a226c1487a08c0b6885122416c14689ee747a2", "cases": 45, "reps": 2, "arms": [ { "id": "jev", "name": "Jev 1.13", "color": "#be542f", "run": "main-jev-v1", "slices": { "all": { "n": 90, "correct": 69, "valid": 90, "recall_n": 30, "missed_recall": 12, "unnecessary_recall": 9, "confusion": { "flow": { "flow": 27, "deliberate": 0, "recall": 9, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 24, "recall": 0, "no_decision": 0 }, "recall": { "flow": 11, "deliberate": 1, "recall": 18, "no_decision": 0 } } }, "en": { "n": 30, "correct": 26, "valid": 30, "recall_n": 10, "missed_recall": 2, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 10, "deliberate": 0, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 2, "deliberate": 0, "recall": 8, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 20, "valid": 30, "recall_n": 10, "missed_recall": 6, "unnecessary_recall": 4, "confusion": { "flow": { "flow": 8, "deliberate": 0, "recall": 4, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 5, "deliberate": 1, "recall": 4, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 23, "valid": 30, "recall_n": 10, "missed_recall": 4, "unnecessary_recall": 3, "confusion": { "flow": { "flow": 9, "deliberate": 0, "recall": 3, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 4, "deliberate": 0, "recall": 6, "no_decision": 0 } } } }, "median_ms": 219.514, "p95_ms": 359.214, "incident_affected": false, "estimated_usd_per_1000": 0.04233693333333333, "cost_usage_n": 90 }, { "id": "laya", "name": "Laya multilingual", "color": "#927aac", "run": "main-low-v1", "slices": { "all": { "n": 90, "correct": 36, "valid": 90, "recall_n": 30, "missed_recall": 30, "unnecessary_recall": 0, "confusion": { "flow": { "flow": 36, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 24, "deliberate": 0, "recall": 0, "no_decision": 0 }, "recall": { "flow": 30, "deliberate": 0, "recall": 0, "no_decision": 0 } } }, "en": { "n": 30, "correct": 12, "valid": 30, "recall_n": 10, "missed_recall": 10, "unnecessary_recall": 0, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 8, "deliberate": 0, "recall": 0, "no_decision": 0 }, "recall": { "flow": 10, "deliberate": 0, "recall": 0, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 12, "valid": 30, "recall_n": 10, "missed_recall": 10, "unnecessary_recall": 0, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 8, "deliberate": 0, "recall": 0, "no_decision": 0 }, "recall": { "flow": 10, "deliberate": 0, "recall": 0, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 12, "valid": 30, "recall_n": 10, "missed_recall": 10, "unnecessary_recall": 0, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 8, "deliberate": 0, "recall": 0, "no_decision": 0 }, "recall": { "flow": 10, "deliberate": 0, "recall": 0, "no_decision": 0 } } } }, "median_ms": 2094.926, "p95_ms": 2870.833, "incident_affected": false, "estimated_usd_per_1000": null, "cost_usage_n": null }, { "id": "laya-english", "name": "Laya English", "color": "#a191b2", "run": "main-laya-english-v1", "slices": { "all": { "n": 90, "correct": 29, "valid": 89, "recall_n": 30, "missed_recall": 26, "unnecessary_recall": 12, "confusion": { "flow": { "flow": 16, "deliberate": 14, "recall": 6, "no_decision": 0 }, "deliberate": { "flow": 8, "deliberate": 9, "recall": 6, "no_decision": 1 }, "recall": { "flow": 18, "deliberate": 8, "recall": 4, "no_decision": 0 } } }, "en": { "n": 30, "correct": 11, "valid": 29, "recall_n": 10, "missed_recall": 10, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 6, "deliberate": 4, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 2, "deliberate": 5, "recall": 0, "no_decision": 1 }, "recall": { "flow": 6, "deliberate": 4, "recall": 0, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 8, "valid": 30, "recall_n": 10, "missed_recall": 6, "unnecessary_recall": 8, "confusion": { "flow": { "flow": 2, "deliberate": 6, "recall": 4, "no_decision": 0 }, "deliberate": { "flow": 2, "deliberate": 2, "recall": 4, "no_decision": 0 }, "recall": { "flow": 6, "deliberate": 0, "recall": 4, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 10, "valid": 30, "recall_n": 10, "missed_recall": 10, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 8, "deliberate": 4, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 4, "deliberate": 2, "recall": 2, "no_decision": 0 }, "recall": { "flow": 6, "deliberate": 4, "recall": 0, "no_decision": 0 } } } }, "median_ms": 5191.529, "p95_ms": 8985.034, "incident_affected": true, "estimated_usd_per_1000": null, "cost_usage_n": null }, { "id": "luna-codex-low", "name": "Luna low", "color": "#427143", "run": "main-codex-low-v1", "slices": { "all": { "n": 90, "correct": 78, "valid": 90, "recall_n": 30, "missed_recall": 0, "unnecessary_recall": 12, "confusion": { "flow": { "flow": 24, "deliberate": 0, "recall": 12, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 24, "recall": 0, "no_decision": 0 }, "recall": { "flow": 0, "deliberate": 0, "recall": 30, "no_decision": 0 } } }, "en": { "n": 30, "correct": 26, "valid": 30, "recall_n": 10, "missed_recall": 0, "unnecessary_recall": 4, "confusion": { "flow": { "flow": 8, "deliberate": 0, "recall": 4, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 0, "deliberate": 0, "recall": 10, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 26, "valid": 30, "recall_n": 10, "missed_recall": 0, "unnecessary_recall": 4, "confusion": { "flow": { "flow": 8, "deliberate": 0, "recall": 4, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 0, "deliberate": 0, "recall": 10, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 26, "valid": 30, "recall_n": 10, "missed_recall": 0, "unnecessary_recall": 4, "confusion": { "flow": { "flow": 8, "deliberate": 0, "recall": 4, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 0, "deliberate": 0, "recall": 10, "no_decision": 0 } } } }, "median_ms": 1357.783, "p95_ms": 2917.678, "incident_affected": false, "estimated_usd_per_1000": 0.067878, "cost_usage_n": 90 }, { "id": "luna-codex-medium@main-codex-medium-v2", "name": "Luna medium · first", "color": "#678059", "run": "main-codex-medium-v2", "slices": { "all": { "n": 90, "correct": 78, "valid": 89, "recall_n": 30, "missed_recall": 4, "unnecessary_recall": 6, "confusion": { "flow": { "flow": 32, "deliberate": 1, "recall": 3, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 20, "recall": 3, "no_decision": 1 }, "recall": { "flow": 4, "deliberate": 0, "recall": 26, "no_decision": 0 } } }, "en": { "n": 30, "correct": 25, "valid": 30, "recall_n": 10, "missed_recall": 2, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 10, "deliberate": 1, "recall": 1, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 7, "recall": 1, "no_decision": 0 }, "recall": { "flow": 2, "deliberate": 0, "recall": 8, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 28, "valid": 30, "recall_n": 10, "missed_recall": 0, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 10, "deliberate": 0, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 0, "deliberate": 0, "recall": 10, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 25, "valid": 29, "recall_n": 10, "missed_recall": 2, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 5, "recall": 2, "no_decision": 1 }, "recall": { "flow": 2, "deliberate": 0, "recall": 8, "no_decision": 0 } } } }, "median_ms": 2153.967, "p95_ms": 5505.593, "incident_affected": false, "estimated_usd_per_1000": 0.097226, "cost_usage_n": 89 }, { "id": "luna-codex-medium@main-codex-medium-v3", "name": "Luna medium · repeat", "color": "#89997b", "run": "main-codex-medium-v3", "slices": { "all": { "n": 90, "correct": 75, "valid": 90, "recall_n": 30, "missed_recall": 5, "unnecessary_recall": 10, "confusion": { "flow": { "flow": 30, "deliberate": 0, "recall": 6, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 20, "recall": 4, "no_decision": 0 }, "recall": { "flow": 5, "deliberate": 0, "recall": 25, "no_decision": 0 } } }, "en": { "n": 30, "correct": 27, "valid": 30, "recall_n": 10, "missed_recall": 1, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 10, "deliberate": 0, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 8, "recall": 0, "no_decision": 0 }, "recall": { "flow": 1, "deliberate": 0, "recall": 9, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 24, "valid": 30, "recall_n": 10, "missed_recall": 2, "unnecessary_recall": 4, "confusion": { "flow": { "flow": 10, "deliberate": 0, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 6, "recall": 2, "no_decision": 0 }, "recall": { "flow": 2, "deliberate": 0, "recall": 8, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 24, "valid": 30, "recall_n": 10, "missed_recall": 2, "unnecessary_recall": 4, "confusion": { "flow": { "flow": 10, "deliberate": 0, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 6, "recall": 2, "no_decision": 0 }, "recall": { "flow": 2, "deliberate": 0, "recall": 8, "no_decision": 0 } } } }, "median_ms": 2106.152, "p95_ms": 6041.613, "incident_affected": false, "estimated_usd_per_1000": 0.100161, "cost_usage_n": 90 }, { "id": "luna-codex-high", "name": "Luna high", "color": "#243e31", "run": "main-codex-high-v1", "slices": { "all": { "n": 90, "correct": 81, "valid": 90, "recall_n": 30, "missed_recall": 3, "unnecessary_recall": 6, "confusion": { "flow": { "flow": 33, "deliberate": 0, "recall": 3, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 21, "recall": 3, "no_decision": 0 }, "recall": { "flow": 3, "deliberate": 0, "recall": 27, "no_decision": 0 } } }, "en": { "n": 30, "correct": 29, "valid": 30, "recall_n": 10, "missed_recall": 0, "unnecessary_recall": 1, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 7, "recall": 1, "no_decision": 0 }, "recall": { "flow": 0, "deliberate": 0, "recall": 10, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 25, "valid": 30, "recall_n": 10, "missed_recall": 2, "unnecessary_recall": 3, "confusion": { "flow": { "flow": 10, "deliberate": 0, "recall": 2, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 7, "recall": 1, "no_decision": 0 }, "recall": { "flow": 2, "deliberate": 0, "recall": 8, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 27, "valid": 30, "recall_n": 10, "missed_recall": 1, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 11, "deliberate": 0, "recall": 1, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 7, "recall": 1, "no_decision": 0 }, "recall": { "flow": 1, "deliberate": 0, "recall": 9, "no_decision": 0 } } } }, "median_ms": 2755.837, "p95_ms": 7119.028, "incident_affected": false, "estimated_usd_per_1000": 0.111356, "cost_usage_n": 90 }, { "id": "sol-codex-low", "name": "Sol low", "color": "#43798b", "run": "main-sol-low-v1", "slices": { "all": { "n": 90, "correct": 74, "valid": 89, "recall_n": 30, "missed_recall": 10, "unnecessary_recall": 5, "confusion": { "flow": { "flow": 36, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 18, "recall": 5, "no_decision": 1 }, "recall": { "flow": 10, "deliberate": 0, "recall": 20, "no_decision": 0 } } }, "en": { "n": 30, "correct": 24, "valid": 29, "recall_n": 10, "missed_recall": 3, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 5, "recall": 2, "no_decision": 1 }, "recall": { "flow": 3, "deliberate": 0, "recall": 7, "no_decision": 0 } } }, "uk": { "n": 30, "correct": 25, "valid": 30, "recall_n": 10, "missed_recall": 3, "unnecessary_recall": 2, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 6, "recall": 2, "no_decision": 0 }, "recall": { "flow": 3, "deliberate": 0, "recall": 7, "no_decision": 0 } } }, "mixed": { "n": 30, "correct": 25, "valid": 30, "recall_n": 10, "missed_recall": 4, "unnecessary_recall": 1, "confusion": { "flow": { "flow": 12, "deliberate": 0, "recall": 0, "no_decision": 0 }, "deliberate": { "flow": 0, "deliberate": 7, "recall": 1, "no_decision": 0 }, "recall": { "flow": 4, "deliberate": 0, "recall": 6, "no_decision": 0 } } } }, "median_ms": 3355.907, "p95_ms": 6557.053, "incident_affected": true, "estimated_usd_per_1000": 1.455618, "cost_usage_n": 89 } ], "notes": [ "Not a general model leaderboard.", "90 attempts per arm are 45 cases repeated twice; translations are correlated.", "All-attempt accuracy includes non-decisions. Recall miss denominator is 30 overall or 10 in each language.", "Latency is successful-decision roundtrip; runs sequential, not head-to-head.", "English Laya and Sol timings affected by client memory pressure; not a clean speed comparison.", "Costs are usage-based/API-equivalent estimates, not observed subscription charges. Unknown usage is not zero.", "Laya local hardware/power cost unknown, not plotted at zero.", "Both usable medium replicates retained; defective medium and historical OpenRouter runs excluded." ] }