{"before":{"method":"cadence-personal-4","checked_at":"2026-10-12T14:10:21.408262+00:00","revision":"dba3af222833c22b9219a84bc155182c1facdf5ad10d1b334a998d4e7fde0ee2","sources":[{"id":"epoch","ok":true,"checked_at":"2026-10-12T14:10:09.307116+00:00","updated_at":"2026-10-12T14:10:09.307116+00:00"},{"id":"writing-comparison","ok":true,"checked_at":"2026-10-12T14:10:09.307982+00:00","updated_at":"2026-10-12T14:10:09.307982+00:00"},{"id":"eq-writing","ok":true,"checked_at":"2026-10-12T14:10:09.308455+00:00","updated_at":"2026-10-12T14:10:09.308455+00:00"},{"id":"eq-longform","ok":true,"checked_at":"2026-10-12T14:10:09.308940+00:00","updated_at":"2026-10-12T14:10:09.308940+00:00"},{"id":"roboflow","ok":true,"checked_at":"2026-10-12T14:10:09.436098+00:00","updated_at":"2026-10-12T14:10:09.436098+00:00"},{"id":"blueprint","ok":true,"checked_at":"2026-10-12T14:10:09.684340+00:00","updated_at":"2026-10-12T14:10:09.684340+00:00"},{"id":"gdp-pdf","ok":true,"checked_at":"2026-10-12T14:10:09.738653+00:00","updated_at":"2026-10-12T14:10:09.738653+00:00"},{"id":"terminal-4","ok":true,"checked_at":"2026-10-12T14:10:09.778658+00:00","updated_at":"2026-10-12T14:10:09.778658+00:00"},{"id":"terminal-science","ok":true,"checked_at":"2026-10-12T14:10:09.780036+00:00","updated_at":"2026-10-12T14:10:09.780036+00:00"},{"id":"weirdml","ok":true,"checked_at":"2026-10-12T14:10:09.933649+00:00","updated_at":"2026-10-12T14:10:09.933649+00:00"},{"id":"livebench","ok":true,"checked_at":"2026-10-12T14:10:10.036788+00:00","updated_at":"2026-10-12T14:10:10.036788+00:00"},{"id":"extractbench","ok":true,"checked_at":"2026-10-12T14:10:10.155699+00:00","updated_at":"2026-10-12T14:10:10.155699+00:00"},{"id":"automationbench","ok":true,"checked_at":"2026-10-12T14:10:10.353313+00:00","updated_at":"2026-10-12T14:10:10.353313+00:00"}],"capabilities":[{"id":"code","name":"Coding","icon":"code","color":"#9ab8e8","scope":"Build software, fix bugs, and work across a codebase.","benchmarks":[{"id":"swe-bench-verified","name":"SWE-Bench verified","source":"Epoch AI","models":32,"latest_model_date":"2026-06-13","url":"https://epoch.ai/benchmarks"},{"id":"aider-polyglot","name":"Aider polyglot","source":"Epoch AI","models":55,"latest_model_date":"2025-12-01","url":"https://epoch.ai/benchmarks"},{"id":"deepswe","name":"DeepSWE","source":"Epoch AI","models":26,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"frontiercode","name":"FrontierCode","source":"Epoch AI","models":42,"latest_model_date":"2026-09-29","url":"https://epoch.ai/benchmarks"},{"id":"gso-bench","name":"GSO-Bench","source":"Epoch AI","models":35,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"mirrorcode","name":"MirrorCode","source":"Epoch AI","models":9,"latest_model_date":"2026-09-22","url":"https://epoch.ai/benchmarks"},{"id":"livebench-code-20260625","name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","models":65,"latest_model_date":"2026-10-07","url":"https://livebench.ai/"}],"limited_models":86,"models":149},{"id":"writing","name":"Writing","icon":"pen","color":"#cfacd4","scope":"Shape ideas into clear, engaging prose.","benchmarks":[{"id":"eq-writing","name":"EQ-Bench Creative Writing v3","source":"EQ-Bench","models":140,"latest_model_date":"2026-09-22","url":"https://eqbench.com/creative_writing.html"},{"id":"eq-longform","name":"EQ-Bench Longform Writing","source":"EQ-Bench","models":140,"latest_model_date":"2026-09-22","url":"https://eqbench.com/creative_writing_longform.html"},{"id":"writing-comparison","name":"Lech Mazur Writing comparison","source":"Lech Mazur","models":53,"latest_model_date":"2026-10-06","url":"https://github.com/lechmazur/writing"},{"id":"livebench-writing-20260625","name":"LiveBench: Instruction Following","source":"LiveBench","models":65,"latest_model_date":"2026-10-07","url":"https://livebench.ai/"}],"note":"Creative prose, storytelling, and following constraints when rewriting or summarizing. Brand voice and factual accuracy need your own evaluation.","limited_models":123,"models":228},{"id":"reading","name":"Reading & documents","icon":"book","color":"#b8cfa0","scope":"Understand documents and extract the details you need.","benchmarks":[{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":35,"latest_model_date":"2026-09-30","url":"https://surgehq.ai/benchmarks/gdp-pdf"},{"id":"roboflow-reading","name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","models":48,"latest_model_date":"2026-10-07","url":"https://playground.roboflow.com/evals"},{"id":"livebench-reading-20260625","name":"LiveBench: Language + Data Analysis","source":"LiveBench","models":65,"latest_model_date":"2026-10-07","url":"https://livebench.ai/"},{"id":"extractbench","name":"ExtractBench: document extraction","source":"LlamaIndex","models":27,"latest_model_date":"2026-09-10","url":"https://github.com/run-llama/ExtractBench"}],"note":"PDF reasoning, document extraction, language comprehension, and structured tables. Full RAG pipelines, million-token recall, and citation accuracy are not measured.","limited_models":55,"models":100},{"id":"reasoning","name":"Reasoning","icon":"path","color":"#d4be8e","scope":"Work through unfamiliar problems and connect the clues.","benchmarks":[{"id":"simplebench","name":"SimpleBench","source":"Epoch AI","models":89,"latest_model_date":"2026-08-12","url":"https://epoch.ai/benchmarks"},{"id":"arc-agi-2","name":"ARC-AGI-2","source":"Epoch AI","models":99,"latest_model_date":"2026-09-29","url":"https://epoch.ai/benchmarks"},{"id":"chess-puzzles","name":"Chess Puzzles","source":"Epoch AI","models":149,"latest_model_date":"2026-09-29","url":"https://epoch.ai/benchmarks"},{"id":"mystery-game-puzzles","name":"Mystery Game Puzzles","source":"Epoch AI","models":82,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"livebench-reasoning-20260625","name":"LiveBench: Reasoning","source":"LiveBench","models":65,"latest_model_date":"2026-10-07","url":"https://livebench.ai/"}],"limited_models":133,"models":247},{"id":"math","name":"Math","icon":"math","color":"#8fc9c8","scope":"Solve quantitative problems, from competitions to proofs.","benchmarks":[{"id":"otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","source":"Epoch AI","models":210,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tiers-1-3-v2-private","name":"FrontierMath-Tiers-1-3-v2-Private","source":"Epoch AI","models":84,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tier-4-v2-private","name":"FrontierMath-Tier-4-v2-Private","source":"Epoch AI","models":66,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"proofbench","name":"ProofBench","source":"Epoch AI","models":78,"latest_model_date":"2026-09-30","url":"https://epoch.ai/benchmarks"},{"id":"livebench-math-20260625","name":"LiveBench: Mathematics","source":"LiveBench","models":65,"latest_model_date":"2026-10-07","url":"https://livebench.ai/"}],"limited_models":158,"models":251},{"id":"science","name":"Science","icon":"science","color":"#a7bbe0","scope":"Apply scientific knowledge to demanding questions and problems.","benchmarks":[{"id":"gpqa-diamond","name":"GPQA diamond","source":"Epoch AI","models":229,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"hle","name":"HLE","source":"Epoch AI","models":45,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"terminal-science","name":"Terminal-Bench Science 0.1","source":"Terminal-Bench Science","models":17,"latest_model_date":"2026-09-22","url":"https://www.terminal-bench-science.ai/"},{"id":"weirdml-v2","name":"WeirdML v2","source":"Håvard Tveit Ihle","models":133,"latest_model_date":"2026-09-04","url":"https://htihle.github.io/weirdml.html"}],"note":"Includes graduate science, cross-subject expert knowledge, research workflows, and machine-learning experiments. It isn't a measure of factual reliability in every field.","limited_models":211,"models":303},{"id":"vision","name":"Visual reasoning","icon":"eye","color":"#d3a78d","scope":"Read visual clues in photographs and diagrams.","benchmarks":[{"id":"roboflow-vision","name":"Vision Evals (low effort)","source":"Roboflow","models":48,"latest_model_date":"2026-10-07","url":"https://playground.roboflow.com/evals"},{"id":"blueprint-2","name":"Blueprint-Bench 2","source":"Andon Labs","models":33,"latest_model_date":"2026-10-07","url":"https://andonlabs.com/evals/blueprint-bench-2"},{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":35,"latest_model_date":"2026-09-30","url":"https://surgehq.ai/benchmarks/gdp-pdf"}],"note":"Image understanding, spatial reconstruction, and visual document reasoning. This does not measure image generation or video.","limited_models":25,"models":62},{"id":"computer","name":"Computer use","icon":"cursor","color":"#b9afdf","scope":"Complete tasks using software, files, and a terminal.","benchmarks":[{"id":"terminal-4","name":"Terminal-Bench 4.0","source":"Terminal-Bench","models":23,"latest_model_date":"2026-09-29","url":"https://www.tbench.ai/leaderboard/terminal-bench/4.0"},{"id":"apex-agents","name":"APEX-Agents","source":"Epoch AI","models":49,"latest_model_date":"2026-09-30","url":"https://epoch.ai/benchmarks"},{"id":"automationbench-1-0-6","name":"AutomationBench 1.0.6","source":"Zapier","models":52,"latest_model_date":"2026-09-30","url":"https://zapier.com/benchmarks"}],"note":"Terminal tasks, professional app work, and business workflows through APIs. Agent setups differ; scores describe the tested model and tools together.","limited_models":40,"models":76}],"weights":{"coding":{"code":3,"reasoning":1},"documents":{"reading":3,"vision":1}},"views":{"coding":{"leaders":[{"id":"catalog:anthropic/claude-fable-5-1","aliases":["catalog:anthropic/claude-fable-5-1","claudefable5.1","claudefable51"],"name":"Claude Fable 5.1","evidence":{"code":{"index":85.37693812116325,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.5091,"configuration":"claude-fable-5-1_medium","effort":"medium","agent":"claude-code","settings":{"Reasoning effort":"medium"},"date":"2026-09-01","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["claudefable5.1"],"index":68.03375261457094,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.8824,"configuration":"claude-fable-5-1_unknown","effort":"unknown","agent":"OpenHands","settings":{},"date":"2026-09-01","source_url":"https://gso-bench.github.io/index.html","standard_error":null,"aliases":["claudefable5.1"],"index":119.99102949976543,"benchmark":"gso-bench","benchmark_name":"GSO-Bench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.733,"configuration":"claude-fable-5-1_high","effort":"high","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.0917,"aliases":["claudefable5.1"],"index":75.48759977131522,"benchmark":"mirrorcode","benchmark_name":"MirrorCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":76.21783333333333,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"code_generation":90.141,"code_completion":82.609,"javascript":68.182,"typescript":60.0,"python":70.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":77.9953705990014,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":7},"reasoning":{"index":76.32070133989441,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.9,"configuration":"claude-fable-5-1_xhigh","effort":"xhigh","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["claudefable5.1"],"index":78.77746091045526,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.47,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.05016135580465918,"aliases":["claudefable5.1"],"index":74.91579991789908,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.58,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.04960449637488583,"aliases":["claudefable5.1"],"index":83.60692947155106,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":91.69225,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":86.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":67.98261505967226,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":5}},"catalog_id":"anthropic/claude-fable-5-1","open_weights":false,"date":"2026-09-01","score":83.11,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"code":{"index":63.08680440944839,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":0.6851441241685144,"configuration":"kimi-k3_max","effort":"max","agent":"mini-swe-agent","settings":{"Reasoning effort":"max"},"date":"2026-07-16","source_url":"https://deepswe.datacurve.ai/","standard_error":null,"aliases":["kimik3"],"index":60.91224242239238,"benchmark":"deepswe","benchmark_name":"DeepSWE","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.44170000000000004,"configuration":"kimi-k3_unknown","effort":"none","agent":"mini-swe-agent","settings":{"Reasoning effort":"none"},"date":"2026-07-16","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["kimik3"],"index":60.90403108581636,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":71.80858333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"code_generation":80.282,"code_completion":82.609,"javascript":68.182,"typescript":53.333,"python":65.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":67.44413972013643,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":7},"reasoning":{"index":62.32661310136515,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":0.6070000000000001,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://lmcouncil.ai/benchmarks","standard_error":null,"aliases":["kimik3"],"index":60.15506828496221,"benchmark":"simplebench","benchmark_name":"SimpleBench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.6041666666666666,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["kimik3"],"index":65.06290661387612,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.39,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":0.04902071300001973,"aliases":["kimik3"],"index":68.0086381014468,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.26,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":0.0440844002276808,"aliases":["kimik3"],"index":52.68696600343634,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":90.673,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"theory_of_mind":82.692,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":80.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":65.71948650310426,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":5,"total":5}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":62.9,"rank":10,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}]},"documents":{"leaders":[{"id":"catalog:openai/gpt-6-astra","aliases":["catalog:openai/gpt-6-astra","gpt6astra"],"name":"GPT-6 Astra","evidence":{"reading":{"index":71.78436629189781,"limited":false,"components":[{"id":"gpt6astra","name":"GPT-6 Astra","score":34.2,"configuration":"GPT 6 Astra (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6astra"],"index":82.25169189261652,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":90.275,"configuration":"gpt-6-astra:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-04T21:25:13Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6 Astra"},"subscores":{"ocr":91.89,"extraction":88.66},"aliases":["gpt6astra"],"index":57.886738396852564,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-08T09:28:00Z"},{"id":"gpt6astra","name":"GPT-6 Astra","score":86.202,"configuration":"gpt-6-astra-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":86.292,"typos":82.0,"consecutive_events":90.555,"tablejoin":58.365,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6astra"],"index":77.69123331539761,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":91.91,"configuration":"OpenAI GPT-6 Astra","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":97.22,"Medium (11–50 pages) F1":90.56,"Long (>50 pages) F1":31.7},"aliases":["gpt6astra"],"index":69.30780156272453,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":78.10782643426187,"limited":false,"components":[{"id":"gpt6astra","name":"GPT-6 Astra","score":86.59666666666666,"configuration":"gpt-6-astra:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-05T00:59:26Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6 Astra"},"subscores":{"ocr":91.89,"extraction":88.66,"counting":80.18,"identification":89.58,"reasoning":87.2,"detection":82.07},"aliases":["gpt6astra"],"index":75.26997423008186,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-08T09:28:00Z"},{"id":"gpt6astra","name":"GPT-6 Astra","score":0.497,"configuration":"GPT-6 Astra","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["gpt6astra"],"index":76.80181318008724,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":34.2,"configuration":"GPT 6 Astra (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6astra"],"index":82.25169189261652,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"openai/gpt-6-astra","open_weights":false,"date":"2026-09-04","score":73.37,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"reading":{"index":58.11538925262577,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":19.0,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":52.800665762384504,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"kimik3","name":"Kimi K3","score":88.77000000000001,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-23T23:46:57Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54},"aliases":["kimik3"],"index":52.20359634536896,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-08T09:28:00Z"},{"id":"kimik3","name":"Kimi K3","score":82.13083333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":72.584,"typos":84.0,"consecutive_events":89.758,"tablejoin":48.404,"tablereformat":98.039},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":66.53930499003656,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"kimik3","name":"Kimi K3","score":83.17,"configuration":"Kimi K3","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":94.64,"Medium (11–50 pages) F1":69.64,"Long (>50 pages) F1":4.99},"aliases":["kimik3"],"index":60.91798991271306,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":49.33501491382376,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":66.50666666666667,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-25T19:51:00Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54,"counting":45.95,"identification":81.25,"reasoning":42.38,"detection":51.92},"aliases":["kimik3"],"index":38.89508622727415,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-08T09:28:00Z"},{"id":"kimik3","name":"Kimi K3","score":0.295,"configuration":"Kimi K3","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["kimik3"],"index":56.30929275181264,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"kimik3","name":"Kimi K3","score":19.0,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":52.800665762384504,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":55.92,"rank":16,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}]}}},"after":{"method":"cadence-personal-4","checked_at":"2026-10-19T14:10:10.899169+00:00","revision":"7bf8b8bb8bb99ba9ceefaf52c75938b3b1ce07122a8a6c06fd204641c0ec6c4e","sources":[{"id":"epoch","ok":true,"checked_at":"2026-10-19T14:10:09.307531+00:00","updated_at":"2026-10-19T14:10:09.307531+00:00"},{"id":"writing-comparison","ok":true,"checked_at":"2026-10-19T14:10:09.309904+00:00","updated_at":"2026-10-19T14:10:09.309904+00:00"},{"id":"eq-writing","ok":true,"checked_at":"2026-10-19T14:10:09.312924+00:00","updated_at":"2026-10-19T14:10:09.312924+00:00"},{"id":"eq-longform","ok":true,"checked_at":"2026-10-19T14:10:09.314522+00:00","updated_at":"2026-10-19T14:10:09.314522+00:00"},{"id":"roboflow","ok":true,"checked_at":"2026-10-19T14:10:09.456306+00:00","updated_at":"2026-10-19T14:10:09.456306+00:00"},{"id":"blueprint","ok":true,"checked_at":"2026-10-19T14:10:09.623784+00:00","updated_at":"2026-10-19T14:10:09.623784+00:00"},{"id":"gdp-pdf","ok":true,"checked_at":"2026-10-19T14:10:09.797164+00:00","updated_at":"2026-10-19T14:10:09.797164+00:00"},{"id":"terminal-4","ok":true,"checked_at":"2026-10-19T14:10:09.815845+00:00","updated_at":"2026-10-19T14:10:09.815845+00:00"},{"id":"terminal-science","ok":true,"checked_at":"2026-10-19T14:10:09.933998+00:00","updated_at":"2026-10-19T14:10:09.933998+00:00"},{"id":"weirdml","ok":true,"checked_at":"2026-10-19T14:10:10.179112+00:00","updated_at":"2026-10-19T14:10:10.179112+00:00"},{"id":"livebench","ok":false,"checked_at":"2026-10-19T14:10:10.316210+00:00","updated_at":"2026-10-18T16:10:09.908710+00:00","error":"HTTPError"},{"id":"extractbench","ok":true,"checked_at":"2026-10-19T14:10:10.336966+00:00","updated_at":"2026-10-19T14:10:10.336966+00:00"},{"id":"automationbench","ok":true,"checked_at":"2026-10-19T14:10:10.380483+00:00","updated_at":"2026-10-19T14:10:10.380483+00:00"}],"capabilities":[{"id":"code","name":"Coding","icon":"code","color":"#9ab8e8","scope":"Build software, fix bugs, and work across a codebase.","benchmarks":[{"id":"swe-bench-verified","name":"SWE-Bench verified","source":"Epoch AI","models":32,"latest_model_date":"2026-06-13","url":"https://epoch.ai/benchmarks"},{"id":"aider-polyglot","name":"Aider polyglot","source":"Epoch AI","models":55,"latest_model_date":"2025-12-01","url":"https://epoch.ai/benchmarks"},{"id":"deepswe","name":"DeepSWE","source":"Epoch AI","models":26,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"frontiercode","name":"FrontierCode","source":"Epoch AI","models":46,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"gso-bench","name":"GSO-Bench","source":"Epoch AI","models":35,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"mirrorcode","name":"MirrorCode","source":"Epoch AI","models":9,"latest_model_date":"2026-09-22","url":"https://epoch.ai/benchmarks"},{"id":"livebench-code-20260625","name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":86,"models":152},{"id":"writing","name":"Writing","icon":"pen","color":"#cfacd4","scope":"Shape ideas into clear, engaging prose.","benchmarks":[{"id":"eq-writing","name":"EQ-Bench Creative Writing v3","source":"EQ-Bench","models":140,"latest_model_date":"2026-09-22","url":"https://eqbench.com/creative_writing.html"},{"id":"eq-longform","name":"EQ-Bench Longform Writing","source":"EQ-Bench","models":140,"latest_model_date":"2026-09-22","url":"https://eqbench.com/creative_writing_longform.html"},{"id":"writing-comparison","name":"Lech Mazur Writing comparison","source":"Lech Mazur","models":55,"latest_model_date":"2026-10-13","url":"https://github.com/lechmazur/writing"},{"id":"livebench-writing-20260625","name":"LiveBench: Instruction Following","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"note":"Creative prose, storytelling, and following constraints when rewriting or summarizing. Brand voice and factual accuracy need your own evaluation.","limited_models":125,"models":231},{"id":"reading","name":"Reading & documents","icon":"book","color":"#b8cfa0","scope":"Understand documents and extract the details you need.","benchmarks":[{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":39,"latest_model_date":"2026-10-13","url":"https://surgehq.ai/benchmarks/gdp-pdf"},{"id":"roboflow-reading","name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","models":50,"latest_model_date":"2026-10-12","url":"https://playground.roboflow.com/evals"},{"id":"livebench-reading-20260625","name":"LiveBench: Language + Data Analysis","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"},{"id":"extractbench","name":"ExtractBench: document extraction","source":"LlamaIndex","models":27,"latest_model_date":"2026-09-10","url":"https://github.com/run-llama/ExtractBench"}],"note":"PDF reasoning, document extraction, language comprehension, and structured tables. Full RAG pipelines, million-token recall, and citation accuracy are not measured.","limited_models":58,"models":104},{"id":"reasoning","name":"Reasoning","icon":"path","color":"#d4be8e","scope":"Work through unfamiliar problems and connect the clues.","benchmarks":[{"id":"simplebench","name":"SimpleBench","source":"Epoch AI","models":89,"latest_model_date":"2026-08-12","url":"https://epoch.ai/benchmarks"},{"id":"arc-agi-2","name":"ARC-AGI-2","source":"Epoch AI","models":101,"latest_model_date":"2026-09-29","url":"https://epoch.ai/benchmarks"},{"id":"chess-puzzles","name":"Chess Puzzles","source":"Epoch AI","models":92,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"mystery-game-puzzles","name":"Mystery Game Puzzles","source":"Epoch AI","models":85,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"livebench-reasoning-20260625","name":"LiveBench: Reasoning","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":94,"models":201},{"id":"math","name":"Math","icon":"math","color":"#8fc9c8","scope":"Solve quantitative problems, from competitions to proofs.","benchmarks":[{"id":"otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","source":"Epoch AI","models":214,"latest_model_date":"2026-10-15","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tiers-1-3-v2-private","name":"FrontierMath-Tiers-1-3-v2-Private","source":"Epoch AI","models":88,"latest_model_date":"2026-10-15","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tier-4-v2-private","name":"FrontierMath-Tier-4-v2-Private","source":"Epoch AI","models":70,"latest_model_date":"2026-10-15","url":"https://epoch.ai/benchmarks"},{"id":"proofbench","name":"ProofBench","source":"Epoch AI","models":81,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"livebench-math-20260625","name":"LiveBench: Mathematics","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":157,"models":254},{"id":"science","name":"Science","icon":"science","color":"#a7bbe0","scope":"Apply scientific knowledge to demanding questions and problems.","benchmarks":[{"id":"gpqa-diamond","name":"GPQA diamond","source":"Epoch AI","models":232,"latest_model_date":"2026-10-15","url":"https://epoch.ai/benchmarks"},{"id":"hle","name":"HLE","source":"Epoch AI","models":47,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"terminal-science","name":"Terminal-Bench Science 0.1","source":"Terminal-Bench Science","models":17,"latest_model_date":"2026-09-22","url":"https://www.terminal-bench-science.ai/"},{"id":"weirdml-v2","name":"WeirdML v2","source":"Håvard Tveit Ihle","models":133,"latest_model_date":"2026-09-04","url":"https://htihle.github.io/weirdml.html"}],"note":"Includes graduate science, cross-subject expert knowledge, research workflows, and machine-learning experiments. It isn't a measure of factual reliability in every field.","limited_models":212,"models":306},{"id":"vision","name":"Visual reasoning","icon":"eye","color":"#d3a78d","scope":"Read visual clues in photographs and diagrams.","benchmarks":[{"id":"roboflow-vision","name":"Vision Evals (low effort)","source":"Roboflow","models":50,"latest_model_date":"2026-10-12","url":"https://playground.roboflow.com/evals"},{"id":"blueprint-2","name":"Blueprint-Bench 2","source":"Andon Labs","models":33,"latest_model_date":"2026-10-13","url":"https://andonlabs.com/evals/blueprint-bench-2"},{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":39,"latest_model_date":"2026-10-13","url":"https://surgehq.ai/benchmarks/gdp-pdf"}],"note":"Image understanding, spatial reconstruction, and visual document reasoning. This does not measure image generation or video.","limited_models":26,"models":65},{"id":"computer","name":"Computer use","icon":"cursor","color":"#b9afdf","scope":"Complete tasks using software, files, and a terminal.","benchmarks":[{"id":"terminal-4","name":"Terminal-Bench 4.0","source":"Terminal-Bench","models":23,"latest_model_date":"2026-09-29","url":"https://www.tbench.ai/leaderboard/terminal-bench/4.0"},{"id":"apex-agents","name":"APEX-Agents","source":"Epoch AI","models":50,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"automationbench-1-0-6","name":"AutomationBench 1.0.6","source":"Zapier","models":52,"latest_model_date":"2026-10-13","url":"https://zapier.com/benchmarks"}],"note":"Terminal tasks, professional app work, and business workflows through APIs. Agent setups differ; scores describe the tested model and tools together.","limited_models":41,"models":77}],"weights":{"coding":{"code":3,"reasoning":1},"documents":{"reading":3,"vision":1}},"views":{"coding":{"leaders":[{"id":"catalog:anthropic/claude-fable-5-1","aliases":["catalog:anthropic/claude-fable-5-1","claudefable5.1","claudefable51"],"name":"Claude Fable 5.1","evidence":{"code":{"index":85.37693812116325,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.5091,"configuration":"claude-fable-5-1_medium","effort":"medium","agent":"claude-code","settings":{"Reasoning effort":"medium"},"date":"2026-09-01","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["claudefable5.1"],"index":68.03375261457094,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.8824,"configuration":"claude-fable-5-1_unknown","effort":"unknown","agent":"OpenHands","settings":{},"date":"2026-09-01","source_url":"https://gso-bench.github.io/index.html","standard_error":null,"aliases":["claudefable5.1"],"index":119.99102949976543,"benchmark":"gso-bench","benchmark_name":"GSO-Bench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.733,"configuration":"claude-fable-5-1_high","effort":"high","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.0917,"aliases":["claudefable5.1"],"index":75.48759977131522,"benchmark":"mirrorcode","benchmark_name":"MirrorCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":76.21783333333333,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"code_generation":90.141,"code_completion":82.609,"javascript":68.182,"typescript":60.0,"python":70.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":77.9953705990014,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":7},"reasoning":{"index":76.53655014665856,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.9,"configuration":"claude-fable-5-1_xhigh","effort":"xhigh","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["claudefable5.1"],"index":78.77746091045526,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.48,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["claudefable5.1"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.58,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.04960449637488583,"aliases":["claudefable5.1"],"index":83.60692947155106,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":91.69225,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":86.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":67.98261505967226,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":5}},"catalog_id":"anthropic/claude-fable-5-1","open_weights":false,"date":"2026-09-01","score":83.17,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:deepseek/deepseek-v4.1-flash","aliases":["catalog:deepseek/deepseek-v4.1-flash","deepseekv4.1flash"],"name":"DeepSeek V4.1 Flash","evidence":{"code":{"index":72.20686917156539,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.4387,"configuration":"deepseek-v4.1-flash_high","effort":"high","agent":"chisel","settings":{"Reasoning effort":"high"},"date":"2026-09-09","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["deepseekv4.1flash"],"index":60.58668443023381,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":78.65483333333333,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"code_generation":77.465,"code_completion":82.609,"javascript":81.818,"typescript":60.0,"python":90.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":83.82705391289696,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":2,"total":7},"reasoning":{"index":68.15771622327182,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.7291666666666666,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["deepseekv4.1flash"],"index":70.85778871102224,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.48,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["deepseekv4.1flash"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.43,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.0497569851956243,"aliases":["deepseekv4.1flash"],"index":69.11319659587228,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":86.69225,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":98.0,"logic_with_navigation":68.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":56.88068444123716,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":5}},"catalog_id":"deepseek/deepseek-v4.1-flash","open_weights":true,"date":"2026-09-10","score":71.19,"rank":6,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":12,"coverage":100,"missing":[]}]},"documents":{"leaders":[{"id":"catalog:openai/gpt-6.1-sol","aliases":["catalog:openai/gpt-6.1-sol","gpt6.1sol"],"name":"GPT-6.1 Sol","evidence":{"reading":{"index":72.91046220002745,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":89.965,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:05Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97},"aliases":["gpt6.1sol"],"index":56.716124419470546,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":86.39266666666667,"configuration":"gpt-6.1-sol-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":84.378,"typos":86.0,"consecutive_events":90.555,"tablejoin":57.423,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6.1sol"],"index":78.21351628114098,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":4},"vision":{"index":78.5189082432278,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":85.47333333333333,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:29Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97,"counting":78.83,"identification":89.58,"reasoning":83.66,"detection":80.84},"aliases":["gpt6.1sol"],"index":73.23607058698475,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":2,"total":3}},"catalog_id":"openai/gpt-6.1-sol","open_weights":false,"date":"2026-09-29","score":74.31,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":4,"benchmark_total":6,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"reading":{"index":58.11538925262577,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":19.0,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":52.800665762384504,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"kimik3","name":"Kimi K3","score":88.77000000000001,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-23T23:46:57Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54},"aliases":["kimik3"],"index":52.20359634536896,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"kimik3","name":"Kimi K3","score":82.13083333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":72.584,"typos":84.0,"consecutive_events":89.758,"tablejoin":48.404,"tablereformat":98.039},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":66.53930499003656,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"kimik3","name":"Kimi K3","score":83.17,"configuration":"Kimi K3","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":94.64,"Medium (11–50 pages) F1":69.64,"Long (>50 pages) F1":4.99},"aliases":["kimik3"],"index":60.91798991271306,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":49.33501491382376,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":66.50666666666667,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-25T19:51:00Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54,"counting":45.95,"identification":81.25,"reasoning":42.38,"detection":51.92},"aliases":["kimik3"],"index":38.89508622727415,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"kimik3","name":"Kimi K3","score":0.295,"configuration":"Kimi K3","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["kimik3"],"index":56.30929275181264,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"kimik3","name":"Kimi K3","score":19.0,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":52.800665762384504,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":55.92,"rank":18,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}]}}},"changes":[{"guide":"coding","lane":"open_leaders","before":[{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"code":{"index":63.08680440944839,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":0.6851441241685144,"configuration":"kimi-k3_max","effort":"max","agent":"mini-swe-agent","settings":{"Reasoning effort":"max"},"date":"2026-07-16","source_url":"https://deepswe.datacurve.ai/","standard_error":null,"aliases":["kimik3"],"index":60.91224242239238,"benchmark":"deepswe","benchmark_name":"DeepSWE","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.44170000000000004,"configuration":"kimi-k3_unknown","effort":"none","agent":"mini-swe-agent","settings":{"Reasoning effort":"none"},"date":"2026-07-16","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["kimik3"],"index":60.90403108581636,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":71.80858333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"code_generation":80.282,"code_completion":82.609,"javascript":68.182,"typescript":53.333,"python":65.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":67.44413972013643,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":7},"reasoning":{"index":62.32661310136515,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":0.6070000000000001,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://lmcouncil.ai/benchmarks","standard_error":null,"aliases":["kimik3"],"index":60.15506828496221,"benchmark":"simplebench","benchmark_name":"SimpleBench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.6041666666666666,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["kimik3"],"index":65.06290661387612,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.39,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":0.04902071300001973,"aliases":["kimik3"],"index":68.0086381014468,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.26,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":0.0440844002276808,"aliases":["kimik3"],"index":52.68696600343634,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":90.673,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"theory_of_mind":82.692,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":80.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":65.71948650310426,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":5,"total":5}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":62.9,"rank":10,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}],"after":[{"id":"catalog:deepseek/deepseek-v4.1-flash","aliases":["catalog:deepseek/deepseek-v4.1-flash","deepseekv4.1flash"],"name":"DeepSeek V4.1 Flash","evidence":{"code":{"index":72.20686917156539,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.4387,"configuration":"deepseek-v4.1-flash_high","effort":"high","agent":"chisel","settings":{"Reasoning effort":"high"},"date":"2026-09-09","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["deepseekv4.1flash"],"index":60.58668443023381,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":78.65483333333333,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"code_generation":77.465,"code_completion":82.609,"javascript":81.818,"typescript":60.0,"python":90.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":83.82705391289696,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":2,"total":7},"reasoning":{"index":68.15771622327182,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.7291666666666666,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["deepseekv4.1flash"],"index":70.85778871102224,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.48,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["deepseekv4.1flash"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.43,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.0497569851956243,"aliases":["deepseekv4.1flash"],"index":69.11319659587228,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":86.69225,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":98.0,"logic_with_navigation":68.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":56.88068444123716,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":5}},"catalog_id":"deepseek/deepseek-v4.1-flash","open_weights":true,"date":"2026-09-10","score":71.19,"rank":6,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":12,"coverage":100,"missing":[]}],"comparisons":[{"previous":"Kimi K3","candidate":"DeepSeek V4.1 Flash","delta":7.45,"tests":6,"current_previous":{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"code":{"index":63.08680440944839,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":0.6851441241685144,"configuration":"kimi-k3_max","effort":"max","agent":"mini-swe-agent","settings":{"Reasoning effort":"max"},"date":"2026-07-16","source_url":"https://deepswe.datacurve.ai/","standard_error":null,"aliases":["kimik3"],"index":60.91224242239238,"benchmark":"deepswe","benchmark_name":"DeepSWE","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.44170000000000004,"configuration":"kimi-k3_unknown","effort":"none","agent":"mini-swe-agent","settings":{"Reasoning effort":"none"},"date":"2026-07-16","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["kimik3"],"index":60.90403108581636,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":71.80858333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"code_generation":80.282,"code_completion":82.609,"javascript":68.182,"typescript":53.333,"python":65.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":67.44413972013643,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":7},"reasoning":{"index":61.98125501054253,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":0.6070000000000001,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://lmcouncil.ai/benchmarks","standard_error":null,"aliases":["kimik3"],"index":60.15506828496221,"benchmark":"simplebench","benchmark_name":"SimpleBench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.6041666666666666,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["kimik3"],"index":65.06290661387612,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.37,"configuration":"kimi-k3_high","effort":"high","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":0.048523658709390974,"aliases":["kimik3"],"index":66.28184764733373,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":0.26,"configuration":"kimi-k3_max","effort":"max","agent":null,"settings":{},"date":"2026-07-16","source_url":"https://epoch.ai/benchmarks","standard_error":0.0440844002276808,"aliases":["kimik3"],"index":52.68696600343634,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"kimik3","name":"Kimi K3","score":90.673,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"theory_of_mind":82.692,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":80.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":65.71948650310426,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":5,"total":5}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":62.81,"rank":12,"delta":0.0,"shared_delta":0.0,"shared_tests":8,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}}]},{"guide":"documents","lane":"leaders","before":[{"id":"catalog:openai/gpt-6-astra","aliases":["catalog:openai/gpt-6-astra","gpt6astra"],"name":"GPT-6 Astra","evidence":{"reading":{"index":71.78436629189781,"limited":false,"components":[{"id":"gpt6astra","name":"GPT-6 Astra","score":34.2,"configuration":"GPT 6 Astra (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6astra"],"index":82.25169189261652,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":90.275,"configuration":"gpt-6-astra:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-04T21:25:13Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6 Astra"},"subscores":{"ocr":91.89,"extraction":88.66},"aliases":["gpt6astra"],"index":57.886738396852564,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-08T09:28:00Z"},{"id":"gpt6astra","name":"GPT-6 Astra","score":86.202,"configuration":"gpt-6-astra-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":86.292,"typos":82.0,"consecutive_events":90.555,"tablejoin":58.365,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6astra"],"index":77.69123331539761,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":91.91,"configuration":"OpenAI GPT-6 Astra","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":97.22,"Medium (11–50 pages) F1":90.56,"Long (>50 pages) F1":31.7},"aliases":["gpt6astra"],"index":69.30780156272453,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":78.10782643426187,"limited":false,"components":[{"id":"gpt6astra","name":"GPT-6 Astra","score":86.59666666666666,"configuration":"gpt-6-astra:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-05T00:59:26Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6 Astra"},"subscores":{"ocr":91.89,"extraction":88.66,"counting":80.18,"identification":89.58,"reasoning":87.2,"detection":82.07},"aliases":["gpt6astra"],"index":75.26997423008186,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-08T09:28:00Z"},{"id":"gpt6astra","name":"GPT-6 Astra","score":0.497,"configuration":"GPT-6 Astra","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["gpt6astra"],"index":76.80181318008724,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":34.2,"configuration":"GPT 6 Astra (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6astra"],"index":82.25169189261652,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"openai/gpt-6-astra","open_weights":false,"date":"2026-09-04","score":73.37,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}],"after":[{"id":"catalog:openai/gpt-6.1-sol","aliases":["catalog:openai/gpt-6.1-sol","gpt6.1sol"],"name":"GPT-6.1 Sol","evidence":{"reading":{"index":72.91046220002745,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":89.965,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:05Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97},"aliases":["gpt6.1sol"],"index":56.716124419470546,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":86.39266666666667,"configuration":"gpt-6.1-sol-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":84.378,"typos":86.0,"consecutive_events":90.555,"tablejoin":57.423,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6.1sol"],"index":78.21351628114098,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":4},"vision":{"index":78.5189082432278,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":85.47333333333333,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:29Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97,"counting":78.83,"identification":89.58,"reasoning":83.66,"detection":80.84},"aliases":["gpt6.1sol"],"index":73.23607058698475,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":2,"total":3}},"catalog_id":"openai/gpt-6.1-sol","open_weights":false,"date":"2026-09-29","score":74.31,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":4,"benchmark_total":6,"coverage":100,"missing":[]}],"comparisons":[{"previous":"GPT-6 Astra","candidate":"GPT-6.1 Sol","delta":0.16,"tests":4,"current_previous":{"id":"catalog:openai/gpt-6-astra","aliases":["catalog:openai/gpt-6-astra","gpt6astra"],"name":"GPT-6 Astra","evidence":{"reading":{"index":71.78436629189781,"limited":false,"components":[{"id":"gpt6astra","name":"GPT-6 Astra","score":34.2,"configuration":"GPT 6 Astra (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6astra"],"index":82.25169189261652,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":90.275,"configuration":"gpt-6-astra:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-04T21:25:13Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6 Astra"},"subscores":{"ocr":91.89,"extraction":88.66},"aliases":["gpt6astra"],"index":57.886738396852564,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"gpt6astra","name":"GPT-6 Astra","score":86.202,"configuration":"gpt-6-astra-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":86.292,"typos":82.0,"consecutive_events":90.555,"tablejoin":58.365,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6astra"],"index":77.69123331539761,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":91.91,"configuration":"OpenAI GPT-6 Astra","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":97.22,"Medium (11–50 pages) F1":90.56,"Long (>50 pages) F1":31.7},"aliases":["gpt6astra"],"index":69.30780156272453,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":78.10782643426187,"limited":false,"components":[{"id":"gpt6astra","name":"GPT-6 Astra","score":86.59666666666666,"configuration":"gpt-6-astra:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-05T00:59:26Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6 Astra"},"subscores":{"ocr":91.89,"extraction":88.66,"counting":80.18,"identification":89.58,"reasoning":87.2,"detection":82.07},"aliases":["gpt6astra"],"index":75.26997423008186,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-13T10:31:39Z"},{"id":"gpt6astra","name":"GPT-6 Astra","score":0.497,"configuration":"GPT-6 Astra","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["gpt6astra"],"index":76.80181318008724,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"gpt6astra","name":"GPT-6 Astra","score":34.2,"configuration":"GPT 6 Astra (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6astra"],"index":82.25169189261652,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"openai/gpt-6-astra","open_weights":false,"date":"2026-09-04","score":73.37,"rank":2,"delta":0.0,"shared_delta":0.0,"shared_tests":6,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}}]}]}