{"schema":1,"generated_at":"2026-10-25T18:23:54+00:00","method":{"prompt":"sudoku-1","max_attempts":2,"set":"v1","per_tier":6,"engine":"74e5819","beyond_pool":400},"tiers":[{"key":"cozy","name":"Cozy","needs":"Singles only","clues":[31,33,32,31,32,42]},{"key":"gentle","name":"Gentle","needs":"Scanning","clues":[25,30,29,30,25,29]},{"key":"steady","name":"Steady","needs":"Locked candidates","clues":[28,28,32,27,28,27]},{"key":"tricky","name":"Tricky","needs":"Hidden pairs and triples","clues":[26,24,24,27,26,25]},{"key":"devious","name":"Devious","needs":"Wings and fish","clues":[26,26,32,41,34,27]},{"key":"expert","name":"Expert","needs":"Deep wings and fish","clues":[23,28,27,24,23,27]},{"key":"beyond","name":"Beyond","needs":"No human technique finishes it","clues":[21,24,25,24,22,24]}],"puzzles":[{"id":"cozy-0","tier":"cozy","k":0,"clues":31,"puzzle":"5..........3..8..2.8..6974..5.49..1.12.....96.49.76..8.6295..3.9..8..2..........1","search_nodes":51,"n":14,"solved":7},{"id":"cozy-1","tier":"cozy","k":1,"clues":33,"puzzle":"8..1.75..69..3.41..7...46.37..9..3....8.5.9....4..3..1..96......32.8..561..3.5..8","search_nodes":49,"n":9,"solved":6},{"id":"cozy-2","tier":"cozy","k":2,"clues":32,"puzzle":".61593....4.2......23...59......1.244...7...665.8......3...826...6..5.7....46238.","search_nodes":50,"n":9,"solved":7},{"id":"cozy-3","tier":"cozy","k":3,"clues":31,"puzzle":"31.....57..9....8287...36..5..27...3...3.5...9...18..5..67....813...45...4.....91","search_nodes":51,"n":8,"solved":7},{"id":"cozy-4","tier":"cozy","k":4,"clues":32,"puzzle":"7.9.2.6..6..4.........36.1.5.2.9...613.6...954..18.3...7.21.......5.87.4..5...9.1","search_nodes":50,"n":8,"solved":6},{"id":"cozy-5","tier":"cozy","k":5,"clues":42,"puzzle":".85.7.2.6.1..3..793.95.241854..139.7.2...5.81..129..451..3.9..2.9.........742.193","search_nodes":40,"n":8,"solved":6},{"id":"gentle-0","tier":"gentle","k":0,"clues":25,"puzzle":"....6.1.2...4.9....3.....7.1...84.3...4.1.7...7.29...4.8.....4....9.1...5.2.7....","search_nodes":299,"n":14,"solved":7},{"id":"gentle-1","tier":"gentle","k":1,"clues":30,"puzzle":".1..4....2....57.9..6.274..78.3...2..2..1..8......2971..543.8..4..2....6....5..4.","search_nodes":90,"n":9,"solved":7},{"id":"gentle-2","tier":"gentle","k":2,"clues":29,"puzzle":"57294..8.89......7...5.....2.....8...8.765.2...7.....1.....6...7......15.4..71298","search_nodes":125,"n":9,"solved":7},{"id":"gentle-3","tier":"gentle","k":3,"clues":30,"puzzle":"..34.2.1..9..3....1.2.......7...3.6153.6.4.7828.7...9.......9.4....1..8..2.8.71..","search_nodes":69,"n":8,"solved":5},{"id":"gentle-4","tier":"gentle","k":4,"clues":25,"puzzle":"...5...7...4..2...8..4..1.63...1....64..5..32....9...12.8..6..3...8..6...7...1...","search_nodes":235,"n":8,"solved":6},{"id":"gentle-5","tier":"gentle","k":5,"clues":29,"puzzle":"....87..1.2..4....86.5.37..1.....68..4..6..9..98.....2..96.2.38....3..2.3..41....","search_nodes":144,"n":8,"solved":6},{"id":"steady-0","tier":"steady","k":0,"clues":28,"puzzle":"..3..8.97.........9...3..18..14..93.45.....76.87..92..59..4...2.........73.6..4..","search_nodes":169,"n":14,"solved":5},{"id":"steady-1","tier":"steady","k":1,"clues":28,"puzzle":"....5...33...7.9..95.4...2..4...2...63..9..47...6...18.9...8.75..6.2...97...4....","search_nodes":114,"n":9,"solved":6},{"id":"steady-2","tier":"steady","k":2,"clues":32,"puzzle":"4.59...812....1..3.31...5....41.5..9.1..2..6.8..3.41....9...35....6....734...79.8","search_nodes":62,"n":9,"solved":5},{"id":"steady-3","tier":"steady","k":3,"clues":27,"puzzle":"24..........3.5..6..3..9...73.6...814...1...912...4.65...4..9..8..7.1..........37","search_nodes":272,"n":8,"solved":7},{"id":"steady-4","tier":"steady","k":4,"clues":28,"puzzle":".63..2..552.4...9.7.....2......479..4.......6..798......6.....3.7...3.698..6..15.","search_nodes":292,"n":8,"solved":6},{"id":"steady-5","tier":"steady","k":5,"clues":27,"puzzle":"......1.381..7..5..5..29.6.......83....617....64.......8.24..1..4..3..253.2......","search_nodes":300,"n":8,"solved":6},{"id":"tricky-0","tier":"tricky","k":0,"clues":26,"puzzle":".4...16.2...7...4..31.....7..5..6.34......52.....8..7.46..19....58.......795.....","search_nodes":699,"n":14,"solved":7},{"id":"tricky-1","tier":"tricky","k":1,"clues":24,"puzzle":"........13.527.4......4..8....75.....97..3..256.......6.1...3..2....5..6.7..9....","search_nodes":1148,"n":9,"solved":4},{"id":"tricky-2","tier":"tricky","k":2,"clues":24,"puzzle":".4...65..9....1..8..1..2.4....4.8....893...........9..75.6...24.68.........2...5.","search_nodes":254,"n":9,"solved":6},{"id":"tricky-3","tier":"tricky","k":3,"clues":27,"puzzle":"91....5....82..9.72...8..147.9......3..57.4.2...8....1..57....6......74.....6..3.","search_nodes":144,"n":8,"solved":7},{"id":"tricky-4","tier":"tricky","k":4,"clues":26,"puzzle":"8.4...3.....7...8..9.........549..7...9..2.....86...249..31.........7.5.6.7..4.92","search_nodes":253,"n":8,"solved":5},{"id":"tricky-5","tier":"tricky","k":5,"clues":25,"puzzle":"....6..5......9......5..1291.....7..3.98...1.8621.............2.249.......145..7.","search_nodes":182,"n":8,"solved":6},{"id":"devious-0","tier":"devious","k":0,"clues":26,"puzzle":"..3..2..5.9...746....59.78.....5.89..241..5..6........3..9.........2.6.8.5.3.....","search_nodes":275,"n":14,"solved":7},{"id":"devious-1","tier":"devious","k":1,"clues":26,"puzzle":".....4283....95..4.62..3........6.2...74......1..8..76..4.67....785....19........","search_nodes":210,"n":9,"solved":3},{"id":"devious-2","tier":"devious","k":2,"clues":32,"puzzle":".2.9.6..45..14.....84...16.6.1.9..2......5...375..2.9.7..8...194..3..2....25...73","search_nodes":67,"n":9,"solved":6},{"id":"devious-3","tier":"devious","k":3,"clues":41,"puzzle":"7831.5.46.4....5..9..3...2...591738431.4.26...97.6.15..2..918...7..3.....395.426.","search_nodes":54,"n":8,"solved":5},{"id":"devious-4","tier":"devious","k":4,"clues":34,"puzzle":"5.1.....8392.4.1..4.89...2.654..........62.34..31.4.5..463.15...89..6..2....97...","search_nodes":73,"n":8,"solved":7},{"id":"devious-5","tier":"devious","k":5,"clues":27,"puzzle":".8...2137....7.....64..1...47.9...1...3.4...5.5.8....2..7..3.....6..452..3....9..","search_nodes":135,"n":8,"solved":4},{"id":"expert-0","tier":"expert","k":0,"clues":23,"puzzle":".......6..7.........92.3.1...7..6.4..5.7.1...2..4....552......79...6.3......1...4","search_nodes":1121,"n":13,"solved":5},{"id":"expert-1","tier":"expert","k":1,"clues":28,"puzzle":"...1.9...1.8.4.6.29......786.....9....761...4.357....1.......1..7....3..5.4.63.8.","search_nodes":97,"n":9,"solved":7},{"id":"expert-2","tier":"expert","k":2,"clues":27,"puzzle":"9....2...5....4..1..3.........8....47.5.3.2..4.1.2..5.2476....9.6..7..151...4....","search_nodes":598,"n":9,"solved":7},{"id":"expert-3","tier":"expert","k":3,"clues":24,"puzzle":"..9.1625......4..9......1...72.....3.......6.6....9..7.3.......4...7.6855..9...4.","search_nodes":456,"n":8,"solved":7},{"id":"expert-4","tier":"expert","k":4,"clues":23,"puzzle":"217...6.....4.1....58....7.7.93.2...5....4.....2....6.............98.1.5..61.....","search_nodes":68,"n":8,"solved":5},{"id":"expert-5","tier":"expert","k":5,"clues":27,"puzzle":"293.7.6.55.....1...1...3..8..9.....3....15....5.4.........462....1.3.7..82.9..3..","search_nodes":118,"n":8,"solved":7},{"id":"beyond-0","tier":"beyond","k":0,"clues":21,"puzzle":"15......2........5...7..4......6..8..2.19....389..5.........2........3917...8....","search_nodes":7411,"n":13,"solved":5},{"id":"beyond-1","tier":"beyond","k":1,"clues":24,"puzzle":"1....4..9..46.......37..6........93..3..7..2....4.8....5..6.1......8...32.1....65","search_nodes":6951,"n":9,"solved":4},{"id":"beyond-2","tier":"beyond","k":2,"clues":25,"puzzle":"......1....4..8..6.6...3.2.......25.8.2..6.9.....5...7..613..8.3...8.7..52...7...","search_nodes":5383,"n":9,"solved":6},{"id":"beyond-3","tier":"beyond","k":3,"clues":24,"puzzle":".....8...261..3...3.8...1...4.89....5...7..3..1..3......4.2...7.....62...7.4....1","search_nodes":4992,"n":8,"solved":6},{"id":"beyond-4","tier":"beyond","k":4,"clues":22,"puzzle":".1.....8..3.....4....4...36..6.9...3.....2.....86...1......5.....1..95...9413....","search_nodes":3932,"n":8,"solved":6},{"id":"beyond-5","tier":"beyond","k":5,"clues":24,"puzzle":"...9.1...2.87......1...6......6.....356.1..4...9..3..7.2...4.3..3..7..18......2..","search_nodes":3672,"n":8,"solved":5}],"contestants":[{"id":"claude-fable-5-1-medium","model_id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","lab":"anthropic","provider":"claude","route":"claude-fable-5-1","effort":"medium","released":"2026-09-01","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":39,"solve_rate":0.929,"score":92.9,"score_low":83.1,"score_high":100.0,"tiers":{"cozy":{"n":6,"solved":6,"rate":1.0,"tokens_out":5128,"seconds":55.5},"gentle":{"n":6,"solved":6,"rate":1.0,"tokens_out":10788,"seconds":108.8},"steady":{"n":6,"solved":6,"rate":1.0,"tokens_out":16626,"seconds":169.7},"tricky":{"n":6,"solved":4,"rate":0.667,"tokens_out":11143,"seconds":121.7},"devious":{"n":6,"solved":5,"rate":0.833,"tokens_out":9188,"seconds":93.4},"expert":{"n":6,"solved":6,"rate":1.0,"tokens_out":23410,"seconds":250.5},"beyond":{"n":6,"solved":6,"rate":1.0,"tokens_out":42711,"seconds":412.9}},"cell_accuracy":0.9758,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.9931,"cells":432},"2":{"name":"Scanning","accuracy":0.9941,"cells":674},"3":{"name":"Locked candidates","accuracy":1.0,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":0.8256,"cells":258},"5":{"name":"Wings and fish","accuracy":0.9879,"cells":165},"6":{"name":"Deep wings and fish","accuracy":1.0,"cells":181},"7":{"name":"Search","accuracy":1.0,"cells":249}},"outcomes":{"solved":39,"wrong":2,"incomplete":0,"changed givens":1,"no grid":0},"rule_breaking":2,"near_misses":2,"confidence_given":1.0,"confidence_mean":96.7,"confidence_when_unsolved":67.7,"confident_and_wrong":2,"brier":0.0469,"retries":1,"truncated_rate":0.0,"cost_known":true,"cost_total":37.5579,"cost_per_puzzle":0.8942,"cost_per_solve":0.963,"tokens_in_per_puzzle":2235,"tokens_out_per_puzzle":16999,"tokens_reasoning_per_puzzle":15035,"seconds_per_puzzle":173.2,"last_attempt":"2026-10-15T21:20:11+00:00","attempt_list":[{"id":"sudoku-6d2863982be2","tier":"beyond","k":5,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":48825,"cost":2.4594700000000005,"finished_at":"2026-10-15T21:20:11+00:00"},{"id":"sudoku-95b54955ce7b","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":21017,"cost":1.06903,"finished_at":"2026-10-15T21:13:12+00:00"},{"id":"sudoku-b72010428470","tier":"devious","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":9839,"cost":0.51023,"finished_at":"2026-10-15T21:08:08+00:00"},{"id":"sudoku-f1901f3b5caf","tier":"steady","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":27190,"cost":1.3774799999999998,"finished_at":"2026-10-15T21:04:42+00:00"},{"id":"sudoku-7ca4a1ce1f40","tier":"tricky","k":5,"outcome":"changed givens","correct":6,"blanks":56,"confidence":5,"tokens_out":879,"cost":0.06187,"finished_at":"2026-10-15T21:04:17+00:00"},{"id":"sudoku-7c6dda40c8cf","tier":"gentle","k":5,"outcome":"solved","correct":52,"blanks":52,"confidence":99,"tokens_out":9360,"cost":0.48624000000000006,"finished_at":"2026-10-15T21:00:21+00:00"},{"id":"sudoku-3763b4528c97","tier":"beyond","k":4,"outcome":"solved","correct":59,"blanks":59,"confidence":97,"tokens_out":35274,"cost":1.78176,"finished_at":"2026-10-15T20:59:03+00:00"},{"id":"sudoku-80efa124e8f2","tier":"cozy","k":5,"outcome":"solved","correct":39,"blanks":39,"confidence":99,"tokens_out":3644,"cost":0.20049999999999998,"finished_at":"2026-10-15T20:57:15+00:00"},{"id":"sudoku-94d638f5e249","tier":"expert","k":4,"outcome":"solved","correct":58,"blanks":58,"confidence":99,"tokens_out":19958,"cost":1.01598,"finished_at":"2026-10-15T20:54:42+00:00"},{"id":"sudoku-8804c47cf615","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":99,"tokens_out":7648,"cost":0.40062000000000003,"finished_at":"2026-10-15T20:50:28+00:00"},{"id":"sudoku-4d20355b5ac2","tier":"tricky","k":4,"outcome":"wrong","correct":53,"blanks":55,"confidence":99,"tokens_out":12423,"cost":0.63935,"finished_at":"2026-10-15T20:50:18+00:00"},{"id":"sudoku-a65fa96a7a61","tier":"steady","k":4,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":9572,"cost":0.49676000000000003,"finished_at":"2026-10-15T20:46:57+00:00"},{"id":"sudoku-393889aeef53","tier":"gentle","k":4,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":12705,"cost":0.65357,"finished_at":"2026-10-15T20:45:24+00:00"},{"id":"sudoku-feb186ec8b55","tier":"cozy","k":4,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":5080,"cost":0.27244,"finished_at":"2026-10-15T20:42:41+00:00"},{"id":"sudoku-250d0904f9ee","tier":"beyond","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":18866,"cost":0.9615,"finished_at":"2026-10-15T20:41:48+00:00"},{"id":"sudoku-7d7a293495ce","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":17430,"cost":0.88958,"finished_at":"2026-10-15T20:38:32+00:00"},{"id":"sudoku-fa6990fd0e4c","tier":"devious","k":3,"outcome":"wrong","correct":38,"blanks":40,"confidence":99,"tokens_out":5441,"cost":0.29029,"finished_at":"2026-10-15T20:35:38+00:00"},{"id":"sudoku-9fc1d352c942","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":12679,"cost":0.65205,"finished_at":"2026-10-15T20:33:35+00:00"},{"id":"sudoku-4a1693cf37c9","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":98,"tokens_out":27307,"cost":1.38343,"finished_at":"2026-10-15T20:33:17+00:00"},{"id":"sudoku-9c6dc22715ad","tier":"gentle","k":3,"outcome":"solved","correct":51,"blanks":51,"confidence":99,"tokens_out":7561,"cost":0.39645,"finished_at":"2026-10-15T20:28:46+00:00"},{"id":"sudoku-76d130e333f4","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":99,"tokens_out":4992,"cost":0.26766,"finished_at":"2026-10-15T20:26:49+00:00"},{"id":"sudoku-64b68cf42a8b","tier":"beyond","k":2,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":51505,"cost":2.5934700000000004,"finished_at":"2026-10-15T20:26:05+00:00"},{"id":"sudoku-f6b4cce39ff0","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":97,"tokens_out":43571,"cost":2.19685,"finished_at":"2026-10-15T20:23:45+00:00"},{"id":"sudoku-dbb5322774b0","tier":"devious","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":8315,"cost":0.43411,"finished_at":"2026-10-15T20:15:18+00:00"},{"id":"sudoku-ba7446a5031e","tier":"tricky","k":2,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":15031,"cost":0.7696299999999999,"finished_at":"2026-10-15T20:13:52+00:00"},{"id":"sudoku-439a284bca0d","tier":"steady","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":14410,"cost":0.73886,"finished_at":"2026-10-15T20:11:26+00:00"},{"id":"sudoku-a074d699e533","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":99,"tokens_out":7537,"cost":0.39485000000000003,"finished_at":"2026-10-15T20:09:02+00:00"},{"id":"sudoku-e074b398d075","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":5501,"cost":0.29307000000000005,"finished_at":"2026-10-15T20:06:58+00:00"},{"id":"sudoku-fe0957c8b1af","tier":"beyond","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":23019,"cost":1.16907,"finished_at":"2026-10-15T20:05:51+00:00"},{"id":"sudoku-ff86786b1584","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":10177,"cost":0.5271100000000001,"finished_at":"2026-10-15T20:02:16+00:00"},{"id":"sudoku-1a85f4b301bb","tier":"devious","k":1,"outcome":"solved","correct":55,"blanks":55,"confidence":99,"tokens_out":13552,"cost":0.69552,"finished_at":"2026-10-15T19:59:14+00:00"},{"id":"sudoku-3a15c9c7ea5a","tier":"tricky","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":10700,"cost":0.5530599999999999,"finished_at":"2026-10-15T19:55:42+00:00"},{"id":"sudoku-036ef40d9db8","tier":"beyond","k":0,"outcome":"solved","correct":60,"blanks":60,"confidence":99,"tokens_out":78775,"cost":5.05263125,"finished_at":"2026-10-15T19:53:09+00:00"},{"id":"sudoku-8dfe8899a63a","tier":"steady","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":9998,"cost":0.51812,"finished_at":"2026-10-15T19:53:07+00:00"},{"id":"sudoku-4b2a1d6ec19b","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":99,"tokens_out":14829,"cost":0.7597100000000001,"finished_at":"2026-10-15T19:51:07+00:00"},{"id":"sudoku-0c6e5ecc2d9c","tier":"cozy","k":1,"outcome":"solved","correct":48,"blanks":48,"confidence":99,"tokens_out":4617,"cost":0.24934999999999996,"finished_at":"2026-10-15T19:47:12+00:00"},{"id":"sudoku-9ed36b9f2c97","tier":"expert","k":0,"outcome":"solved","correct":58,"blanks":58,"confidence":97,"tokens_out":28310,"cost":1.43366,"finished_at":"2026-10-15T19:41:48+00:00"},{"id":"sudoku-765f20dce620","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":99,"tokens_out":10330,"cost":0.5347400000000001,"finished_at":"2026-10-15T19:37:04+00:00"},{"id":"sudoku-185a7475d972","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":15148,"cost":0.7754,"finished_at":"2026-10-15T19:35:21+00:00"},{"id":"sudoku-32fb4340c3e9","tier":"steady","k":0,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":11281,"cost":0.58209,"finished_at":"2026-10-15T19:32:30+00:00"},{"id":"sudoku-a6ed01b4f523","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":12739,"cost":0.65535,"finished_at":"2026-10-15T19:30:20+00:00"},{"id":"sudoku-e9edcd1205fd","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":6935,"cost":0.36495,"finished_at":"2026-10-15T19:28:21+00:00"}]},{"id":"claude-haiku-5-5-medium","model_id":"anthropic/claude-haiku-5-5","name":"Claude Haiku 5.5","lab":"anthropic","provider":"claude","route":"claude-haiku-5-5","effort":"medium","released":"2026-10-07","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":13,"solve_rate":0.31,"score":31.0,"score_low":18.7,"score_high":43.2,"tiers":{"cozy":{"n":6,"solved":3,"rate":0.5,"tokens_out":15853,"seconds":59.7},"gentle":{"n":6,"solved":1,"rate":0.167,"tokens_out":22690,"seconds":82.8},"steady":{"n":6,"solved":1,"rate":0.167,"tokens_out":34434,"seconds":127.0},"tricky":{"n":6,"solved":3,"rate":0.5,"tokens_out":33600,"seconds":126.2},"devious":{"n":6,"solved":2,"rate":0.333,"tokens_out":20784,"seconds":76.9},"expert":{"n":6,"solved":2,"rate":0.333,"tokens_out":38906,"seconds":145.5},"beyond":{"n":6,"solved":1,"rate":0.167,"tokens_out":44833,"seconds":161.8}},"cell_accuracy":0.931,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.963,"cells":432},"2":{"name":"Scanning","accuracy":0.9392,"cells":674},"3":{"name":"Locked candidates","accuracy":0.9307,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":0.9147,"cells":258},"5":{"name":"Wings and fish","accuracy":0.9212,"cells":165},"6":{"name":"Deep wings and fish","accuracy":0.9006,"cells":181},"7":{"name":"Search","accuracy":0.8996,"cells":249}},"outcomes":{"solved":13,"wrong":8,"incomplete":0,"changed givens":21,"no grid":0},"rule_breaking":8,"near_misses":7,"confidence_given":1.0,"confidence_mean":96.5,"confidence_when_unsolved":96.4,"confident_and_wrong":29,"brier":0.6427,"retries":0,"truncated_rate":0.0,"cost_known":true,"cost_total":0.6406,"cost_per_puzzle":0.0153,"cost_per_solve":0.0493,"tokens_in_per_puzzle":912,"tokens_out_per_puzzle":30157,"tokens_reasoning_per_puzzle":30053,"seconds_per_puzzle":111.4,"last_attempt":"2026-10-15T21:15:19+00:00","attempt_list":[{"id":"sudoku-e220cc7efa30","tier":"beyond","k":5,"outcome":"changed givens","correct":49,"blanks":57,"confidence":96,"tokens_out":41784,"cost":0.0210744,"finished_at":"2026-10-15T21:15:19+00:00"},{"id":"sudoku-90af826ff45e","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":32847,"cost":0.0166061,"finished_at":"2026-10-15T21:11:36+00:00"},{"id":"sudoku-b5c8565d211c","tier":"devious","k":5,"outcome":"wrong","correct":52,"blanks":54,"confidence":97,"tokens_out":21335,"cost":0.010850700000000001,"finished_at":"2026-10-15T21:08:14+00:00"},{"id":"sudoku-b255e54c2e66","tier":"tricky","k":5,"outcome":"solved","correct":56,"blanks":56,"confidence":96,"tokens_out":34925,"cost":0.017641699999999996,"finished_at":"2026-10-15T21:06:25+00:00"},{"id":"sudoku-36feabb54ecd","tier":"steady","k":5,"outcome":"changed givens","correct":45,"blanks":54,"confidence":97,"tokens_out":34036,"cost":0.017198399999999996,"finished_at":"2026-10-15T21:02:26+00:00"},{"id":"sudoku-13dab3f86519","tier":"gentle","k":5,"outcome":"changed givens","correct":45,"blanks":52,"confidence":95,"tokens_out":16733,"cost":0.008549300000000001,"finished_at":"2026-10-15T21:00:07+00:00"},{"id":"sudoku-cc83012f580f","tier":"cozy","k":5,"outcome":"changed givens","correct":37,"blanks":39,"confidence":96,"tokens_out":13730,"cost":0.007048399999999999,"finished_at":"2026-10-15T20:58:07+00:00"},{"id":"sudoku-ac15fafe0967","tier":"beyond","k":4,"outcome":"solved","correct":59,"blanks":59,"confidence":96,"tokens_out":38412,"cost":0.0193866,"finished_at":"2026-10-15T20:56:34+00:00"},{"id":"sudoku-57189e535f59","tier":"expert","k":4,"outcome":"changed givens","correct":56,"blanks":58,"confidence":96,"tokens_out":27271,"cost":0.0138163,"finished_at":"2026-10-15T20:53:04+00:00"},{"id":"sudoku-fbf415dc98a6","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":97,"tokens_out":18428,"cost":0.0093966,"finished_at":"2026-10-15T20:51:16+00:00"},{"id":"sudoku-e94632e46eec","tier":"tricky","k":4,"outcome":"changed givens","correct":46,"blanks":55,"confidence":97,"tokens_out":18264,"cost":0.0093146,"finished_at":"2026-10-15T20:49:05+00:00"},{"id":"sudoku-9d219a48bb28","tier":"steady","k":4,"outcome":"changed givens","correct":53,"blanks":53,"confidence":97,"tokens_out":36937,"cost":0.0186501,"finished_at":"2026-10-15T20:47:54+00:00"},{"id":"sudoku-40101451844c","tier":"gentle","k":4,"outcome":"wrong","correct":55,"blanks":56,"confidence":97,"tokens_out":37740,"cost":0.0190534,"finished_at":"2026-10-15T20:45:38+00:00"},{"id":"sudoku-e3b286e3ecab","tier":"cozy","k":4,"outcome":"changed givens","correct":47,"blanks":49,"confidence":96,"tokens_out":14485,"cost":0.0074272999999999995,"finished_at":"2026-10-15T20:43:20+00:00"},{"id":"sudoku-0a18292f60c9","tier":"beyond","k":3,"outcome":"wrong","correct":55,"blanks":57,"confidence":96,"tokens_out":37645,"cost":0.019004899999999998,"finished_at":"2026-10-15T20:40:55+00:00"},{"id":"sudoku-4b675756cf1b","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":97,"tokens_out":43745,"cost":0.022053899999999998,"finished_at":"2026-10-15T20:38:25+00:00"},{"id":"sudoku-0725f44039b3","tier":"devious","k":3,"outcome":"solved","correct":40,"blanks":40,"confidence":97,"tokens_out":14358,"cost":0.007362,"finished_at":"2026-10-15T20:35:36+00:00"},{"id":"sudoku-9b99152ca00a","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":97,"tokens_out":27394,"cost":0.013878800000000002,"finished_at":"2026-10-15T20:34:53+00:00"},{"id":"sudoku-74f312f108d3","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":40368,"cost":0.0203656,"finished_at":"2026-10-15T20:31:25+00:00"},{"id":"sudoku-4675b40db6a6","tier":"gentle","k":3,"outcome":"wrong","correct":49,"blanks":51,"confidence":96,"tokens_out":19744,"cost":0.010056200000000001,"finished_at":"2026-10-15T20:28:41+00:00"},{"id":"sudoku-828383c210c6","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":97,"tokens_out":22664,"cost":0.011513200000000001,"finished_at":"2026-10-15T20:27:30+00:00"},{"id":"sudoku-3e98b2cbe9e3","tier":"beyond","k":2,"outcome":"wrong","correct":54,"blanks":56,"confidence":97,"tokens_out":55398,"cost":0.0278822,"finished_at":"2026-10-15T20:22:25+00:00"},{"id":"sudoku-53bd744748b4","tier":"expert","k":2,"outcome":"changed givens","correct":54,"blanks":54,"confidence":97,"tokens_out":26482,"cost":0.0134246,"finished_at":"2026-10-15T20:17:33+00:00"},{"id":"sudoku-ded535b00a87","tier":"devious","k":2,"outcome":"changed givens","correct":44,"blanks":49,"confidence":96,"tokens_out":20278,"cost":0.010323200000000001,"finished_at":"2026-10-15T20:15:39+00:00"},{"id":"sudoku-4cc50c245d6c","tier":"tricky","k":2,"outcome":"solved","correct":57,"blanks":57,"confidence":96,"tokens_out":40672,"cost":0.0205176,"finished_at":"2026-10-15T20:13:41+00:00"},{"id":"sudoku-558e529cb335","tier":"steady","k":2,"outcome":"wrong","correct":48,"blanks":49,"confidence":97,"tokens_out":20374,"cost":0.010371,"finished_at":"2026-10-15T20:10:12+00:00"},{"id":"sudoku-4681d8e83aed","tier":"gentle","k":2,"outcome":"wrong","correct":50,"blanks":52,"confidence":97,"tokens_out":17928,"cost":0.0091442,"finished_at":"2026-10-15T20:08:55+00:00"},{"id":"sudoku-65abdd4eee2a","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":97,"tokens_out":14523,"cost":0.007442099999999999,"finished_at":"2026-10-15T20:07:49+00:00"},{"id":"sudoku-a2e9766695c1","tier":"beyond","k":1,"outcome":"changed givens","correct":48,"blanks":57,"confidence":95,"tokens_out":44178,"cost":0.0222704,"finished_at":"2026-10-15T20:04:53+00:00"},{"id":"sudoku-484d5ff1c64f","tier":"expert","k":1,"outcome":"changed givens","correct":36,"blanks":53,"confidence":97,"tokens_out":19760,"cost":0.0100628,"finished_at":"2026-10-15T20:01:45+00:00"},{"id":"sudoku-80ca23b5ab6d","tier":"devious","k":1,"outcome":"changed givens","correct":46,"blanks":55,"confidence":95,"tokens_out":30128,"cost":0.0152436,"finished_at":"2026-10-15T19:59:06+00:00"},{"id":"sudoku-1894197742b8","tier":"tricky","k":1,"outcome":"changed givens","correct":51,"blanks":57,"confidence":97,"tokens_out":45939,"cost":0.0231503,"finished_at":"2026-10-15T19:57:16+00:00"},{"id":"sudoku-1675b71daaf7","tier":"steady","k":1,"outcome":"changed givens","correct":45,"blanks":53,"confidence":97,"tokens_out":26957,"cost":0.0136609,"finished_at":"2026-10-15T19:53:46+00:00"},{"id":"sudoku-bfbc1e2b879a","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":97,"tokens_out":19228,"cost":0.0097968,"finished_at":"2026-10-15T19:50:23+00:00"},{"id":"sudoku-0c016eab6ac9","tier":"cozy","k":1,"outcome":"changed givens","correct":45,"blanks":48,"confidence":97,"tokens_out":12834,"cost":0.006602599999999999,"finished_at":"2026-10-15T19:47:58+00:00"},{"id":"sudoku-fb18a220fd1a","tier":"beyond","k":0,"outcome":"changed givens","correct":52,"blanks":60,"confidence":97,"tokens_out":51581,"cost":0.025863419999999998,"finished_at":"2026-10-15T19:46:21+00:00"},{"id":"sudoku-01c1c2829934","tier":"expert","k":0,"outcome":"changed givens","correct":52,"blanks":58,"confidence":97,"tokens_out":83330,"cost":0.04173814,"finished_at":"2026-10-15T19:43:10+00:00"},{"id":"sudoku-874bbcad8ff5","tier":"devious","k":0,"outcome":"changed givens","correct":48,"blanks":55,"confidence":96,"tokens_out":20177,"cost":0.0102707,"finished_at":"2026-10-15T19:36:52+00:00"},{"id":"sudoku-16dfc3cc2f22","tier":"tricky","k":0,"outcome":"changed givens","correct":42,"blanks":55,"confidence":96,"tokens_out":34408,"cost":0.0173848,"finished_at":"2026-10-15T19:35:36+00:00"},{"id":"sudoku-f5084d44dae0","tier":"steady","k":0,"outcome":"wrong","correct":47,"blanks":53,"confidence":96,"tokens_out":47932,"cost":0.0241468,"finished_at":"2026-10-15T19:33:30+00:00"},{"id":"sudoku-4362d27d23d3","tier":"gentle","k":0,"outcome":"changed givens","correct":50,"blanks":56,"confidence":97,"tokens_out":24768,"cost":0.0125678,"finished_at":"2026-10-15T19:29:49+00:00"},{"id":"sudoku-62d59431f8fe","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":97,"tokens_out":16880,"cost":0.00851316,"finished_at":"2026-10-15T19:28:13+00:00"}]},{"id":"claude-opus-5-5-medium","model_id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","lab":"anthropic","provider":"claude","route":"claude-opus-5-5","effort":"medium","released":"2026-09-22","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":42,"solve_rate":1.0,"score":100.0,"score_low":91.3,"score_high":100.0,"tiers":{"cozy":{"n":6,"solved":6,"rate":1.0,"tokens_out":4521,"seconds":40.7},"gentle":{"n":6,"solved":6,"rate":1.0,"tokens_out":8296,"seconds":70.7},"steady":{"n":6,"solved":6,"rate":1.0,"tokens_out":12067,"seconds":101.8},"tricky":{"n":6,"solved":6,"rate":1.0,"tokens_out":16782,"seconds":142.8},"devious":{"n":6,"solved":6,"rate":1.0,"tokens_out":8398,"seconds":74.0},"expert":{"n":6,"solved":6,"rate":1.0,"tokens_out":11761,"seconds":101.6},"beyond":{"n":6,"solved":6,"rate":1.0,"tokens_out":20962,"seconds":174.3}},"cell_accuracy":1.0,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":1.0,"cells":432},"2":{"name":"Scanning","accuracy":1.0,"cells":674},"3":{"name":"Locked candidates","accuracy":1.0,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":1.0,"cells":258},"5":{"name":"Wings and fish","accuracy":1.0,"cells":165},"6":{"name":"Deep wings and fish","accuracy":1.0,"cells":181},"7":{"name":"Search","accuracy":1.0,"cells":249}},"outcomes":{"solved":42,"wrong":0,"incomplete":0,"changed givens":0,"no grid":0},"rule_breaking":0,"near_misses":0,"confidence_given":1.0,"confidence_mean":98.8,"confidence_when_unsolved":null,"confident_and_wrong":0,"brier":0.0002,"retries":0,"truncated_rate":0.0,"cost_known":true,"cost_total":10.2311,"cost_per_puzzle":0.2436,"cost_per_solve":0.2436,"tokens_in_per_puzzle":910,"tokens_out_per_puzzle":11827,"tokens_reasoning_per_puzzle":11720,"seconds_per_puzzle":100.8,"last_attempt":"2026-10-15T21:16:49+00:00","attempt_list":[{"id":"sudoku-cc5c60898242","tier":"beyond","k":5,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":25750,"cost":0.522288,"finished_at":"2026-10-15T21:16:49+00:00"},{"id":"sudoku-8796eb384de9","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":98,"tokens_out":8959,"cost":0.186476,"finished_at":"2026-10-15T21:11:51+00:00"},{"id":"sudoku-e8db642a4a2a","tier":"devious","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":9224,"cost":0.191792,"finished_at":"2026-10-15T21:09:30+00:00"},{"id":"sudoku-250efd54d944","tier":"tricky","k":5,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":16603,"cost":0.33922800000000003,"finished_at":"2026-10-15T21:06:57+00:00"},{"id":"sudoku-a1597ba09635","tier":"steady","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":22349,"cost":0.45418000000000003,"finished_at":"2026-10-15T21:04:02+00:00"},{"id":"sudoku-56d78c468fea","tier":"gentle","k":5,"outcome":"solved","correct":52,"blanks":52,"confidence":99,"tokens_out":6795,"cost":0.143188,"finished_at":"2026-10-15T21:00:06+00:00"},{"id":"sudoku-20f100a4baa0","tier":"cozy","k":5,"outcome":"solved","correct":39,"blanks":39,"confidence":97,"tokens_out":2821,"cost":0.06374,"finished_at":"2026-10-15T20:58:35+00:00"},{"id":"sudoku-59a9b278ebfc","tier":"beyond","k":4,"outcome":"solved","correct":59,"blanks":59,"confidence":99,"tokens_out":13090,"cost":0.269016,"finished_at":"2026-10-15T20:56:31+00:00"},{"id":"sudoku-446c45bf5fb3","tier":"expert","k":4,"outcome":"solved","correct":58,"blanks":58,"confidence":99,"tokens_out":11675,"cost":0.240724,"finished_at":"2026-10-15T20:53:12+00:00"},{"id":"sudoku-4b99f84f1215","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":98,"tokens_out":7356,"cost":0.15440800000000002,"finished_at":"2026-10-15T20:51:25+00:00"},{"id":"sudoku-9c395339d50a","tier":"tricky","k":4,"outcome":"solved","correct":55,"blanks":55,"confidence":99,"tokens_out":7272,"cost":0.15272,"finished_at":"2026-10-15T20:49:14+00:00"},{"id":"sudoku-550bd2b145c0","tier":"steady","k":4,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":11004,"cost":0.227336,"finished_at":"2026-10-15T20:47:47+00:00"},{"id":"sudoku-8c6bdafbda3f","tier":"gentle","k":4,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":9477,"cost":0.196868,"finished_at":"2026-10-15T20:44:42+00:00"},{"id":"sudoku-5880aacc7b30","tier":"cozy","k":4,"outcome":"solved","correct":49,"blanks":49,"confidence":98,"tokens_out":4713,"cost":0.101636,"finished_at":"2026-10-15T20:43:20+00:00"},{"id":"sudoku-e8061dfc845c","tier":"beyond","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":14079,"cost":0.28886,"finished_at":"2026-10-15T20:42:24+00:00"},{"id":"sudoku-ab233c4921e8","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":98,"tokens_out":14467,"cost":0.29658000000000007,"finished_at":"2026-10-15T20:38:42+00:00"},{"id":"sudoku-c6458888f8e1","tier":"devious","k":3,"outcome":"solved","correct":40,"blanks":40,"confidence":98,"tokens_out":4464,"cost":0.096592,"finished_at":"2026-10-15T20:35:35+00:00"},{"id":"sudoku-5404a5442976","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":98,"tokens_out":9107,"cost":0.18938800000000003,"finished_at":"2026-10-15T20:34:39+00:00"},{"id":"sudoku-3f4af34a02a6","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":13500,"cost":0.27723200000000003,"finished_at":"2026-10-15T20:31:23+00:00"},{"id":"sudoku-c9abf941fbd8","tier":"gentle","k":3,"outcome":"solved","correct":51,"blanks":51,"confidence":99,"tokens_out":6695,"cost":0.14126800000000003,"finished_at":"2026-10-15T20:28:53+00:00"},{"id":"sudoku-fe00dcf04ec4","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":98,"tokens_out":4224,"cost":0.091704,"finished_at":"2026-10-15T20:27:28+00:00"},{"id":"sudoku-b07edc845b82","tier":"beyond","k":2,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":26961,"cost":0.54654,"finished_at":"2026-10-15T20:25:58+00:00"},{"id":"sudoku-1b06eecb697e","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":99,"tokens_out":9845,"cost":0.204236,"finished_at":"2026-10-15T20:17:42+00:00"},{"id":"sudoku-1ea2edc03f93","tier":"devious","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":6056,"cost":0.12847199999999998,"finished_at":"2026-10-15T20:15:57+00:00"},{"id":"sudoku-b91ebfdebe98","tier":"tricky","k":2,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":19667,"cost":0.40058800000000006,"finished_at":"2026-10-15T20:14:21+00:00"},{"id":"sudoku-0012686dbabe","tier":"steady","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":6845,"cost":0.14423600000000003,"finished_at":"2026-10-15T20:11:08+00:00"},{"id":"sudoku-c80b5084b9eb","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":99,"tokens_out":7432,"cost":0.15584,"finished_at":"2026-10-15T20:08:54+00:00"},{"id":"sudoku-eca19dead1a4","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":99,"tokens_out":4891,"cost":0.105028,"finished_at":"2026-10-15T20:07:43+00:00"},{"id":"sudoku-b212e1a686d3","tier":"beyond","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":30356,"cost":0.6099142,"finished_at":"2026-10-15T20:06:59+00:00"},{"id":"sudoku-4ca02f0b79b6","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":6811,"cost":0.143516,"finished_at":"2026-10-15T20:01:38+00:00"},{"id":"sudoku-731091a3a77e","tier":"devious","k":1,"outcome":"solved","correct":55,"blanks":55,"confidence":99,"tokens_out":11121,"cost":0.22958800000000001,"finished_at":"2026-10-15T20:00:37+00:00"},{"id":"sudoku-544e271fa4dd","tier":"tricky","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":99,"tokens_out":38164,"cost":0.770504,"finished_at":"2026-10-15T20:00:31+00:00"},{"id":"sudoku-20e8e6a1bfc0","tier":"steady","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":8562,"cost":0.178536,"finished_at":"2026-10-15T19:54:22+00:00"},{"id":"sudoku-e3594c91cdbe","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":99,"tokens_out":7892,"cost":0.165128,"finished_at":"2026-10-15T19:51:28+00:00"},{"id":"sudoku-34d4deac08d4","tier":"cozy","k":1,"outcome":"solved","correct":48,"blanks":48,"confidence":98,"tokens_out":4320,"cost":0.0938,"finished_at":"2026-10-15T19:48:37+00:00"},{"id":"sudoku-c1043f98f990","tier":"beyond","k":0,"outcome":"solved","correct":60,"blanks":60,"confidence":99,"tokens_out":15536,"cost":0.3135024,"finished_at":"2026-10-15T19:45:36+00:00"},{"id":"sudoku-96e04c374f92","tier":"expert","k":0,"outcome":"solved","correct":58,"blanks":58,"confidence":99,"tokens_out":18807,"cost":0.383404,"finished_at":"2026-10-15T19:40:53+00:00"},{"id":"sudoku-0dcc0c70ecd5","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":99,"tokens_out":12170,"cost":0.25068,"finished_at":"2026-10-15T19:38:18+00:00"},{"id":"sudoku-e88438997b09","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":99,"tokens_out":9881,"cost":0.204836,"finished_at":"2026-10-15T19:34:59+00:00"},{"id":"sudoku-41b7d4cd9df1","tier":"steady","k":0,"outcome":"solved","correct":53,"blanks":53,"confidence":99,"tokens_out":10144,"cost":0.210096,"finished_at":"2026-10-15T19:32:47+00:00"},{"id":"sudoku-b4cdb6055f09","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":99,"tokens_out":11483,"cost":0.237012,"finished_at":"2026-10-15T19:30:25+00:00"},{"id":"sudoku-2cf2d55879e0","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":99,"tokens_out":6155,"cost":0.130388,"finished_at":"2026-10-15T19:28:05+00:00"}]},{"id":"claude-sonnet-5-5-medium","model_id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","lab":"anthropic","provider":"claude","route":"claude-sonnet-5-5","effort":"medium","released":"2026-09-28","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":37,"solve_rate":0.881,"score":88.1,"score_low":77.6,"score_high":98.6,"tiers":{"cozy":{"n":6,"solved":5,"rate":0.833,"tokens_out":5884,"seconds":42.8},"gentle":{"n":6,"solved":5,"rate":0.833,"tokens_out":10288,"seconds":70.6},"steady":{"n":6,"solved":5,"rate":0.833,"tokens_out":14392,"seconds":96.0},"tricky":{"n":6,"solved":6,"rate":1.0,"tokens_out":17670,"seconds":119.1},"devious":{"n":6,"solved":4,"rate":0.667,"tokens_out":8807,"seconds":62.0},"expert":{"n":6,"solved":6,"rate":1.0,"tokens_out":14058,"seconds":95.1},"beyond":{"n":6,"solved":6,"rate":1.0,"tokens_out":23134,"seconds":150.3}},"cell_accuracy":0.9946,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.9861,"cells":432},"2":{"name":"Scanning","accuracy":0.9955,"cells":674},"3":{"name":"Locked candidates","accuracy":1.0,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":1.0,"cells":258},"5":{"name":"Wings and fish","accuracy":0.9818,"cells":165},"6":{"name":"Deep wings and fish","accuracy":1.0,"cells":181},"7":{"name":"Search","accuracy":1.0,"cells":249}},"outcomes":{"solved":37,"wrong":1,"incomplete":0,"changed givens":4,"no grid":0},"rule_breaking":1,"near_misses":1,"confidence_given":1.0,"confidence_mean":96.4,"confidence_when_unsolved":96.2,"confident_and_wrong":5,"brier":0.1113,"retries":0,"truncated_rate":0.0,"cost_known":true,"cost_total":5.805,"cost_per_puzzle":0.1382,"cost_per_solve":0.1569,"tokens_in_per_puzzle":913,"tokens_out_per_puzzle":13462,"tokens_reasoning_per_puzzle":13332,"seconds_per_puzzle":90.8,"last_attempt":"2026-10-15T21:18:08+00:00","attempt_list":[{"id":"sudoku-bbfa39e3e732","tier":"beyond","k":5,"outcome":"solved","correct":57,"blanks":57,"confidence":96,"tokens_out":25722,"cost":0.260872,"finished_at":"2026-10-15T21:18:08+00:00"},{"id":"sudoku-8b57f10f0b18","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":11023,"cost":0.113886,"finished_at":"2026-10-15T21:12:51+00:00"},{"id":"sudoku-0ceab9d99432","tier":"tricky","k":5,"outcome":"solved","correct":56,"blanks":56,"confidence":97,"tokens_out":49720,"cost":0.500796,"finished_at":"2026-10-15T21:10:32+00:00"},{"id":"sudoku-45b5d06bec56","tier":"devious","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":9252,"cost":0.09619599999999999,"finished_at":"2026-10-15T21:09:20+00:00"},{"id":"sudoku-5b72df71980c","tier":"steady","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":23390,"cost":0.237508,"finished_at":"2026-10-15T21:04:59+00:00"},{"id":"sudoku-057cf04e5550","tier":"gentle","k":5,"outcome":"solved","correct":52,"blanks":52,"confidence":96,"tokens_out":8213,"cost":0.085794,"finished_at":"2026-10-15T21:01:03+00:00"},{"id":"sudoku-4a89bda25d30","tier":"cozy","k":5,"outcome":"solved","correct":39,"blanks":39,"confidence":96,"tokens_out":4079,"cost":0.044469999999999996,"finished_at":"2026-10-15T20:59:08+00:00"},{"id":"sudoku-2d1ea9f72ed6","tier":"beyond","k":4,"outcome":"solved","correct":59,"blanks":59,"confidence":96,"tokens_out":20492,"cost":0.208544,"finished_at":"2026-10-15T20:58:44+00:00"},{"id":"sudoku-9aa776b7021d","tier":"expert","k":4,"outcome":"solved","correct":58,"blanks":58,"confidence":96,"tokens_out":11302,"cost":0.116648,"finished_at":"2026-10-15T20:54:19+00:00"},{"id":"sudoku-c2211ab4594b","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":96,"tokens_out":8397,"cost":0.08762600000000001,"finished_at":"2026-10-15T20:51:28+00:00"},{"id":"sudoku-e258cb18dc78","tier":"tricky","k":4,"outcome":"solved","correct":55,"blanks":55,"confidence":97,"tokens_out":9307,"cost":0.09673,"finished_at":"2026-10-15T20:50:10+00:00"},{"id":"sudoku-c729082c34cd","tier":"steady","k":4,"outcome":"solved","correct":53,"blanks":53,"confidence":97,"tokens_out":11851,"cost":0.122158,"finished_at":"2026-10-15T20:48:13+00:00"},{"id":"sudoku-a759c7126a6f","tier":"gentle","k":4,"outcome":"solved","correct":56,"blanks":56,"confidence":96,"tokens_out":13687,"cost":0.140542,"finished_at":"2026-10-15T20:46:14+00:00"},{"id":"sudoku-6f0a50b4bafb","tier":"cozy","k":4,"outcome":"solved","correct":49,"blanks":49,"confidence":97,"tokens_out":5007,"cost":0.053774,"finished_at":"2026-10-15T20:43:17+00:00"},{"id":"sudoku-31b69c3ad86c","tier":"beyond","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":97,"tokens_out":15752,"cost":0.16117599999999999,"finished_at":"2026-10-15T20:42:39+00:00"},{"id":"sudoku-df198ec844f0","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":96,"tokens_out":16706,"cost":0.170696,"finished_at":"2026-10-15T20:40:22+00:00"},{"id":"sudoku-675325cceb03","tier":"devious","k":3,"outcome":"changed givens","correct":37,"blanks":40,"confidence":96,"tokens_out":8022,"cost":0.083888,"finished_at":"2026-10-15T20:36:34+00:00"},{"id":"sudoku-de95150a1193","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":97,"tokens_out":10031,"cost":0.10395,"finished_at":"2026-10-15T20:34:42+00:00"},{"id":"sudoku-57d8af4f8174","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":15119,"cost":0.154826,"finished_at":"2026-10-15T20:33:05+00:00"},{"id":"sudoku-445309073d2f","tier":"gentle","k":3,"outcome":"solved","correct":51,"blanks":51,"confidence":97,"tokens_out":5971,"cost":0.063402,"finished_at":"2026-10-15T20:29:24+00:00"},{"id":"sudoku-d6f197ede031","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":96,"tokens_out":7940,"cost":0.083028,"finished_at":"2026-10-15T20:27:53+00:00"},{"id":"sudoku-f937c3a769ff","tier":"beyond","k":2,"outcome":"solved","correct":56,"blanks":56,"confidence":97,"tokens_out":31352,"cost":0.31718799999999997,"finished_at":"2026-10-15T20:26:58+00:00"},{"id":"sudoku-240c100d44f6","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":96,"tokens_out":12852,"cost":0.132196,"finished_at":"2026-10-15T20:18:58+00:00"},{"id":"sudoku-dab468e6d171","tier":"devious","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":96,"tokens_out":8835,"cost":0.09203800000000002,"finished_at":"2026-10-15T20:16:18+00:00"},{"id":"sudoku-3ec5a65b32e4","tier":"tricky","k":2,"outcome":"solved","correct":57,"blanks":57,"confidence":97,"tokens_out":11914,"cost":0.12278,"finished_at":"2026-10-15T20:15:01+00:00"},{"id":"sudoku-7d57e8f6af66","tier":"steady","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":96,"tokens_out":9012,"cost":0.093804,"finished_at":"2026-10-15T20:11:11+00:00"},{"id":"sudoku-da4ca4e253ac","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":96,"tokens_out":10182,"cost":0.105428,"finished_at":"2026-10-15T20:10:08+00:00"},{"id":"sudoku-6cf704c816e7","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":97,"tokens_out":6908,"cost":0.0727,"finished_at":"2026-10-15T20:07:47+00:00"},{"id":"sudoku-aaa54cf5697a","tier":"beyond","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":97,"tokens_out":19087,"cost":0.19451,"finished_at":"2026-10-15T20:06:53+00:00"},{"id":"sudoku-78d30a4fb9af","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":96,"tokens_out":10296,"cost":0.106612,"finished_at":"2026-10-15T20:02:52+00:00"},{"id":"sudoku-a85a032e34de","tier":"devious","k":1,"outcome":"wrong","correct":53,"blanks":55,"confidence":96,"tokens_out":8502,"cost":0.088624,"finished_at":"2026-10-15T20:00:15+00:00"},{"id":"sudoku-8e7e6c4c4986","tier":"tricky","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":97,"tokens_out":11653,"cost":0.12014999999999999,"finished_at":"2026-10-15T19:57:00+00:00"},{"id":"sudoku-8b74c187ba58","tier":"steady","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":96,"tokens_out":17456,"cost":0.178212,"finished_at":"2026-10-15T19:55:12+00:00"},{"id":"sudoku-b5186b0fbc0c","tier":"gentle","k":1,"outcome":"changed givens","correct":50,"blanks":51,"confidence":96,"tokens_out":10024,"cost":0.1039,"finished_at":"2026-10-15T19:52:12+00:00"},{"id":"sudoku-89b35a9e44b8","tier":"cozy","k":1,"outcome":"changed givens","correct":43,"blanks":48,"confidence":96,"tokens_out":5223,"cost":0.05594999999999999,"finished_at":"2026-10-15T19:49:15+00:00"},{"id":"sudoku-cf18f2c0a176","tier":"beyond","k":0,"outcome":"solved","correct":60,"blanks":60,"confidence":97,"tokens_out":26396,"cost":0.267524,"finished_at":"2026-10-15T19:48:39+00:00"},{"id":"sudoku-cfaecd5bb30d","tier":"expert","k":0,"outcome":"solved","correct":58,"blanks":58,"confidence":97,"tokens_out":22166,"cost":0.22307340000000003,"finished_at":"2026-10-15T19:43:18+00:00"},{"id":"sudoku-ea51b76d2f19","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":96,"tokens_out":9835,"cost":0.10201,"finished_at":"2026-10-15T19:37:59+00:00"},{"id":"sudoku-6f651113eec2","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":96,"tokens_out":13397,"cost":0.13759,"finished_at":"2026-10-15T19:36:32+00:00"},{"id":"sudoku-8442b4aec2be","tier":"steady","k":0,"outcome":"changed givens","correct":52,"blanks":53,"confidence":97,"tokens_out":9526,"cost":0.09888,"finished_at":"2026-10-15T19:33:34+00:00"},{"id":"sudoku-781ecfcbc2e8","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":97,"tokens_out":13650,"cost":0.14019600000000002,"finished_at":"2026-10-15T19:31:21+00:00"},{"id":"sudoku-dbfb46874282","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":96,"tokens_out":6149,"cost":0.065146,"finished_at":"2026-10-15T19:28:51+00:00"}]},{"id":"codex-gpt-6-astra-medium","model_id":"openai/gpt-6-astra","name":"GPT-6 Astra","lab":"openai","provider":"codex","route":"gpt-6-astra","effort":"medium","released":"2026-09-03","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":32,"solve_rate":0.762,"score":76.2,"score_low":64.4,"score_high":88.0,"tiers":{"cozy":{"n":6,"solved":5,"rate":0.833,"tokens_out":2768,"seconds":95.9},"gentle":{"n":6,"solved":4,"rate":0.667,"tokens_out":4133,"seconds":130.3},"steady":{"n":6,"solved":5,"rate":0.833,"tokens_out":10358,"seconds":325.2},"tricky":{"n":6,"solved":4,"rate":0.667,"tokens_out":5546,"seconds":183.1},"devious":{"n":6,"solved":4,"rate":0.667,"tokens_out":8886,"seconds":296.4},"expert":{"n":6,"solved":6,"rate":1.0,"tokens_out":5917,"seconds":206.8},"beyond":{"n":6,"solved":4,"rate":0.667,"tokens_out":11303,"seconds":344.3}},"cell_accuracy":0.9404,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.9815,"cells":432},"2":{"name":"Scanning","accuracy":0.957,"cells":674},"3":{"name":"Locked candidates","accuracy":0.9818,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":0.8411,"cells":258},"5":{"name":"Wings and fish","accuracy":0.9091,"cells":165},"6":{"name":"Deep wings and fish","accuracy":1.0,"cells":181},"7":{"name":"Search","accuracy":0.8594,"cells":249}},"outcomes":{"solved":32,"wrong":9,"incomplete":0,"changed givens":1,"no grid":0},"rule_breaking":9,"near_misses":0,"confidence_given":1.0,"confidence_mean":100.0,"confidence_when_unsolved":100.0,"confident_and_wrong":10,"brier":0.2381,"retries":0,"truncated_rate":0.0,"cost_known":true,"cost_total":19.0246,"cost_per_puzzle":0.453,"cost_per_solve":0.5945,"tokens_in_per_puzzle":27660,"tokens_out_per_puzzle":6987,"tokens_reasoning_per_puzzle":6614,"seconds_per_puzzle":226.0,"last_attempt":"2026-10-15T21:43:29+00:00","attempt_list":[{"id":"sudoku-1f886ba1db03","tier":"beyond","k":5,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":14610,"cost":0.857996,"finished_at":"2026-10-15T21:43:29+00:00"},{"id":"sudoku-a90b7b9c5110","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":3180,"cost":0.274436,"finished_at":"2026-10-15T21:36:24+00:00"},{"id":"sudoku-ab986f251b9b","tier":"steady","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":28379,"cost":1.490086,"finished_at":"2026-10-15T21:35:48+00:00"},{"id":"sudoku-75d0bdc933bc","tier":"devious","k":5,"outcome":"changed givens","correct":53,"blanks":54,"confidence":100,"tokens_out":4657,"cost":0.32497,"finished_at":"2026-10-15T21:33:05+00:00"},{"id":"sudoku-989299bd2a96","tier":"tricky","k":5,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":4500,"cost":0.282498,"finished_at":"2026-10-15T21:28:18+00:00"},{"id":"sudoku-ba4685897dc2","tier":"gentle","k":5,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":2887,"cost":0.259946,"finished_at":"2026-10-15T21:22:29+00:00"},{"id":"sudoku-0c8edbbe568e","tier":"cozy","k":5,"outcome":"solved","correct":39,"blanks":39,"confidence":100,"tokens_out":2048,"cost":0.137686,"finished_at":"2026-10-15T21:21:02+00:00"},{"id":"sudoku-bcaf05c540a0","tier":"beyond","k":4,"outcome":"solved","correct":59,"blanks":59,"confidence":100,"tokens_out":5950,"cost":0.358712,"finished_at":"2026-10-15T21:19:17+00:00"},{"id":"sudoku-6d5ec12b79e4","tier":"expert","k":4,"outcome":"solved","correct":58,"blanks":58,"confidence":100,"tokens_out":8503,"cost":0.459696,"finished_at":"2026-10-15T21:16:16+00:00"},{"id":"sudoku-02b1bf75c622","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":100,"tokens_out":4585,"cost":0.264106,"finished_at":"2026-10-15T21:13:24+00:00"},{"id":"sudoku-edc845a5eb7c","tier":"tricky","k":4,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":6078,"cost":0.52503,"finished_at":"2026-10-15T21:10:53+00:00"},{"id":"sudoku-949c48ad2bd2","tier":"steady","k":4,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":11426,"cost":0.606056,"finished_at":"2026-10-15T21:10:50+00:00"},{"id":"sudoku-9fc574186eda","tier":"gentle","k":4,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":5908,"cost":0.49979,"finished_at":"2026-10-15T21:04:40+00:00"},{"id":"sudoku-de14a0fd5123","tier":"cozy","k":4,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":1845,"cost":0.208296,"finished_at":"2026-10-15T21:01:35+00:00"},{"id":"sudoku-27820d2d4463","tier":"beyond","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":12496,"cost":0.822776,"finished_at":"2026-10-15T21:01:20+00:00"},{"id":"sudoku-6c6c4b4807c8","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":4921,"cost":0.326364,"finished_at":"2026-10-15T20:54:31+00:00"},{"id":"sudoku-432864583cf1","tier":"devious","k":3,"outcome":"solved","correct":40,"blanks":40,"confidence":100,"tokens_out":3833,"cost":0.356326,"finished_at":"2026-10-15T20:52:29+00:00"},{"id":"sudoku-3488fe54583a","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":6032,"cost":0.3901,"finished_at":"2026-10-15T20:50:04+00:00"},{"id":"sudoku-11c155938436","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":13044,"cost":0.699896,"finished_at":"2026-10-15T20:49:18+00:00"},{"id":"sudoku-f584c84a1cd4","tier":"beyond","k":2,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":20556,"cost":1.087976,"finished_at":"2026-10-15T20:41:32+00:00"},{"id":"sudoku-a18cb01636a9","tier":"gentle","k":3,"outcome":"wrong","correct":33,"blanks":51,"confidence":100,"tokens_out":2681,"cost":0.249966,"finished_at":"2026-10-15T20:41:12+00:00"},{"id":"sudoku-7d3bee46996e","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":2923,"cost":0.261516,"finished_at":"2026-10-15T20:39:00+00:00"},{"id":"sudoku-5487a174f671","tier":"devious","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":15585,"cost":0.965386,"finished_at":"2026-10-15T20:32:07+00:00"},{"id":"sudoku-fac99b616ef7","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":4814,"cost":0.379786,"finished_at":"2026-10-15T20:29:17+00:00"},{"id":"sudoku-c012f72a78fc","tier":"tricky","k":2,"outcome":"wrong","correct":26,"blanks":57,"confidence":100,"tokens_out":5110,"cost":0.341074,"finished_at":"2026-10-15T20:22:58+00:00"},{"id":"sudoku-34fc1da83e1c","tier":"steady","k":2,"outcome":"wrong","correct":43,"blanks":49,"confidence":100,"tokens_out":2359,"cost":0.179302,"finished_at":"2026-10-15T20:19:02+00:00"},{"id":"sudoku-7b8a3a910086","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":3848,"cost":0.342096,"finished_at":"2026-10-15T20:15:38+00:00"},{"id":"sudoku-a60df2c9b860","tier":"cozy","k":2,"outcome":"wrong","correct":41,"blanks":49,"confidence":100,"tokens_out":4855,"cost":0.357856,"finished_at":"2026-10-15T20:13:39+00:00"},{"id":"sudoku-034f191a9b35","tier":"beyond","k":1,"outcome":"wrong","correct":34,"blanks":57,"confidence":100,"tokens_out":7619,"cost":0.466202,"finished_at":"2026-10-15T20:09:55+00:00"},{"id":"sudoku-0e95b5517d8b","tier":"devious","k":1,"outcome":"wrong","correct":40,"blanks":55,"confidence":100,"tokens_out":11490,"cost":0.658314,"finished_at":"2026-10-15T20:07:50+00:00"},{"id":"sudoku-697155a6140b","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":2708,"cost":0.170296,"finished_at":"2026-10-15T20:05:55+00:00"},{"id":"sudoku-72cb4d41a65c","tier":"tricky","k":1,"outcome":"wrong","correct":46,"blanks":57,"confidence":100,"tokens_out":5801,"cost":0.380444,"finished_at":"2026-10-15T20:01:32+00:00"},{"id":"sudoku-efcba3bb02d4","tier":"steady","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":3227,"cost":0.237352,"finished_at":"2026-10-15T19:57:41+00:00"},{"id":"sudoku-44f6aa61f8d2","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":100,"tokens_out":5579,"cost":0.352722,"finished_at":"2026-10-15T19:55:46+00:00"},{"id":"sudoku-d4543a90be14","tier":"cozy","k":1,"outcome":"solved","correct":48,"blanks":48,"confidence":100,"tokens_out":3141,"cost":0.303706,"finished_at":"2026-10-15T19:52:33+00:00"},{"id":"sudoku-08c85e9b3319","tier":"expert","k":0,"outcome":"solved","correct":58,"blanks":58,"confidence":100,"tokens_out":11374,"cost":0.698964,"finished_at":"2026-10-15T19:50:44+00:00"},{"id":"sudoku-4ea40f7d5e2c","tier":"beyond","k":0,"outcome":"wrong","correct":48,"blanks":60,"confidence":100,"tokens_out":6586,"cost":0.376818,"finished_at":"2026-10-15T19:50:08+00:00"},{"id":"sudoku-7cc539cb385b","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":13165,"cost":0.786456,"finished_at":"2026-10-15T19:43:36+00:00"},{"id":"sudoku-6e54b77aeebc","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":5754,"cost":0.473422,"finished_at":"2026-10-15T19:36:01+00:00"},{"id":"sudoku-774f1506a294","tier":"steady","k":0,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":3710,"cost":0.348458,"finished_at":"2026-10-15T19:32:19+00:00"},{"id":"sudoku-77a494ce3ebb","tier":"gentle","k":0,"outcome":"wrong","correct":48,"blanks":56,"confidence":100,"tokens_out":3896,"cost":0.310576,"finished_at":"2026-10-15T19:30:15+00:00"},{"id":"sudoku-0c5bfff41dc1","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":1797,"cost":0.151162,"finished_at":"2026-10-15T19:28:21+00:00"}]},{"id":"codex-gpt-6-luna-medium","model_id":"openai/gpt-6-luna","name":"GPT-6 Luna","lab":"openai","provider":"codex","route":"gpt-6-luna","effort":"medium","released":"2026-09-22","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":0,"solve_rate":0.0,"score":0.0,"score_low":0.0,"score_high":8.7,"tiers":{"cozy":{"n":6,"solved":0,"rate":0.0,"tokens_out":1826,"seconds":23.1},"gentle":{"n":6,"solved":0,"rate":0.0,"tokens_out":1485,"seconds":20.4},"steady":{"n":6,"solved":0,"rate":0.0,"tokens_out":1971,"seconds":27.3},"tricky":{"n":6,"solved":0,"rate":0.0,"tokens_out":1748,"seconds":24.3},"devious":{"n":6,"solved":0,"rate":0.0,"tokens_out":1836,"seconds":23.5},"expert":{"n":6,"solved":0,"rate":0.0,"tokens_out":2203,"seconds":30.4},"beyond":{"n":6,"solved":0,"rate":0.0,"tokens_out":1381,"seconds":21.6}},"cell_accuracy":0.0855,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.1752,"cells":331},"2":{"name":"Scanning","accuracy":0.1285,"cells":498},"3":{"name":"Locked candidates","accuracy":0.1731,"cells":104},"4":{"name":"Hidden pairs and triples","accuracy":0.0513,"cells":156},"5":{"name":"Wings and fish","accuracy":0.1407,"cells":135},"6":{"name":"Deep wings and fish","accuracy":0.0,"cells":54},"7":{"name":"Search","accuracy":0.1206,"cells":199}},"outcomes":{"solved":0,"wrong":3,"incomplete":5,"changed givens":20,"no grid":14},"rule_breaking":3,"near_misses":0,"confidence_given":0.9524,"confidence_mean":7.5,"confidence_when_unsolved":7.5,"confident_and_wrong":3,"brier":0.075,"retries":24,"truncated_rate":0.0,"cost_known":true,"cost_total":0.1016,"cost_per_puzzle":0.0024,"cost_per_solve":null,"tokens_in_per_puzzle":56078,"tokens_out_per_puzzle":1779,"tokens_reasoning_per_puzzle":1234,"seconds_per_puzzle":24.4,"last_attempt":"2026-10-15T21:38:08+00:00","attempt_list":[{"id":"sudoku-1e3d0ffbc131","tier":"beyond","k":5,"outcome":"no grid","correct":0,"blanks":57,"confidence":0,"tokens_out":1901,"cost":0.00351084,"finished_at":"2026-10-15T21:38:08+00:00"},{"id":"sudoku-d6ccb77bdf41","tier":"expert","k":5,"outcome":"no grid","correct":0,"blanks":54,"confidence":0,"tokens_out":2569,"cost":0.0028886,"finished_at":"2026-10-15T21:36:09+00:00"},{"id":"sudoku-5f806132fc0b","tier":"devious","k":5,"outcome":"changed givens","correct":0,"blanks":54,"confidence":0,"tokens_out":1610,"cost":0.00181406,"finished_at":"2026-10-15T21:33:27+00:00"},{"id":"sudoku-23d1c3d2c672","tier":"tricky","k":5,"outcome":"no grid","correct":0,"blanks":56,"confidence":0,"tokens_out":1950,"cost":0.002652,"finished_at":"2026-10-15T21:28:46+00:00"},{"id":"sudoku-dcba785a101a","tier":"steady","k":5,"outcome":"no grid","correct":0,"blanks":54,"confidence":0,"tokens_out":1476,"cost":0.00242456,"finished_at":"2026-10-15T21:24:08+00:00"},{"id":"sudoku-a834aa75f711","tier":"gentle","k":5,"outcome":"no grid","correct":0,"blanks":52,"confidence":null,"tokens_out":2344,"cost":0.0032695,"finished_at":"2026-10-15T21:22:31+00:00"},{"id":"sudoku-60c44cac9eb0","tier":"cozy","k":5,"outcome":"changed givens","correct":12,"blanks":39,"confidence":0,"tokens_out":1583,"cost":0.0012312,"finished_at":"2026-10-15T21:20:17+00:00"},{"id":"sudoku-4a49b5d1bc20","tier":"beyond","k":4,"outcome":"changed givens","correct":7,"blanks":59,"confidence":0,"tokens_out":1367,"cost":0.00157594,"finished_at":"2026-10-15T21:19:15+00:00"},{"id":"sudoku-5548a10412b9","tier":"expert","k":4,"outcome":"incomplete","correct":0,"blanks":58,"confidence":0,"tokens_out":1405,"cost":0.0016190199999999999,"finished_at":"2026-10-15T21:13:44+00:00"},{"id":"sudoku-c7e1a9a15eea","tier":"devious","k":4,"outcome":"wrong","correct":15,"blanks":47,"confidence":0,"tokens_out":1801,"cost":0.00177238,"finished_at":"2026-10-15T21:11:13+00:00"},{"id":"sudoku-c9caeef59165","tier":"tricky","k":4,"outcome":"changed givens","correct":11,"blanks":55,"confidence":0,"tokens_out":2857,"cost":0.00396092,"finished_at":"2026-10-15T21:08:30+00:00"},{"id":"sudoku-8bab8c74c015","tier":"steady","k":4,"outcome":"no grid","correct":0,"blanks":53,"confidence":null,"tokens_out":2571,"cost":0.00278082,"finished_at":"2026-10-15T21:05:19+00:00"},{"id":"sudoku-b225653c95db","tier":"gentle","k":4,"outcome":"wrong","correct":5,"blanks":56,"confidence":0,"tokens_out":2558,"cost":0.00261336,"finished_at":"2026-10-15T21:02:41+00:00"},{"id":"sudoku-13f7568848a6","tier":"cozy","k":4,"outcome":"no grid","correct":0,"blanks":49,"confidence":0,"tokens_out":1932,"cost":0.0035574200000000004,"finished_at":"2026-10-15T21:01:20+00:00"},{"id":"sudoku-85fb71313574","tier":"beyond","k":3,"outcome":"changed givens","correct":12,"blanks":57,"confidence":100,"tokens_out":53,"cost":0.00022520000000000003,"finished_at":"2026-10-15T20:54:52+00:00"},{"id":"sudoku-5a1f1aa9302c","tier":"expert","k":3,"outcome":"no grid","correct":0,"blanks":57,"confidence":0,"tokens_out":1689,"cost":0.00201492,"finished_at":"2026-10-15T20:52:54+00:00"},{"id":"sudoku-7f98e34dbc95","tier":"devious","k":3,"outcome":"wrong","correct":24,"blanks":40,"confidence":0,"tokens_out":1216,"cost":0.00120114,"finished_at":"2026-10-15T20:51:00+00:00"},{"id":"sudoku-2f04c727c6d1","tier":"tricky","k":3,"outcome":"changed givens","correct":0,"blanks":54,"confidence":0,"tokens_out":1142,"cost":0.00238394,"finished_at":"2026-10-15T20:49:32+00:00"},{"id":"sudoku-22e38e221bb2","tier":"steady","k":3,"outcome":"no grid","correct":0,"blanks":54,"confidence":0,"tokens_out":2445,"cost":0.0036349399999999997,"finished_at":"2026-10-15T20:42:44+00:00"},{"id":"sudoku-570b89609c8c","tier":"gentle","k":3,"outcome":"incomplete","correct":0,"blanks":51,"confidence":0,"tokens_out":1480,"cost":0.0017768200000000001,"finished_at":"2026-10-15T20:41:02+00:00"},{"id":"sudoku-29463e4182cc","tier":"cozy","k":3,"outcome":"changed givens","correct":16,"blanks":50,"confidence":0,"tokens_out":2920,"cost":0.00272038,"finished_at":"2026-10-15T20:39:33+00:00"},{"id":"sudoku-86b5bba079aa","tier":"beyond","k":2,"outcome":"changed givens","correct":0,"blanks":56,"confidence":0,"tokens_out":1068,"cost":0.0011753,"finished_at":"2026-10-15T20:32:22+00:00"},{"id":"sudoku-1800eae423b3","tier":"expert","k":2,"outcome":"incomplete","correct":0,"blanks":54,"confidence":0,"tokens_out":1588,"cost":0.00162128,"finished_at":"2026-10-15T20:29:15+00:00"},{"id":"sudoku-03343bf8bf36","tier":"devious","k":2,"outcome":"no grid","correct":0,"blanks":49,"confidence":0,"tokens_out":3058,"cost":0.0030165599999999997,"finished_at":"2026-10-15T20:25:03+00:00"},{"id":"sudoku-11067a8370e0","tier":"tricky","k":2,"outcome":"changed givens","correct":10,"blanks":57,"confidence":0,"tokens_out":1312,"cost":0.0013652,"finished_at":"2026-10-15T20:22:09+00:00"},{"id":"sudoku-d9352a0dd949","tier":"steady","k":2,"outcome":"changed givens","correct":15,"blanks":49,"confidence":0,"tokens_out":905,"cost":0.0021776400000000002,"finished_at":"2026-10-15T20:19:07+00:00"},{"id":"sudoku-467baac73d9b","tier":"gentle","k":2,"outcome":"changed givens","correct":7,"blanks":52,"confidence":100,"tokens_out":53,"cost":0.00022420000000000003,"finished_at":"2026-10-15T20:14:44+00:00"},{"id":"sudoku-f39068306423","tier":"cozy","k":2,"outcome":"changed givens","correct":11,"blanks":49,"confidence":100,"tokens_out":53,"cost":0.0008695199999999999,"finished_at":"2026-10-15T20:11:18+00:00"},{"id":"sudoku-d471f59b2895","tier":"beyond","k":1,"outcome":"changed givens","correct":5,"blanks":57,"confidence":0,"tokens_out":1715,"cost":0.00361462,"finished_at":"2026-10-15T20:08:18+00:00"},{"id":"sudoku-6494197e85ce","tier":"expert","k":1,"outcome":"no grid","correct":0,"blanks":53,"confidence":0,"tokens_out":2533,"cost":0.0041100600000000004,"finished_at":"2026-10-15T20:05:13+00:00"},{"id":"sudoku-1d0034b3ed44","tier":"devious","k":1,"outcome":"changed givens","correct":0,"blanks":55,"confidence":0,"tokens_out":1485,"cost":0.0023537600000000003,"finished_at":"2026-10-15T20:02:05+00:00"},{"id":"sudoku-bc7ea73f5622","tier":"tricky","k":1,"outcome":"no grid","correct":0,"blanks":57,"confidence":0,"tokens_out":2107,"cost":0.00413524,"finished_at":"2026-10-15T19:59:22+00:00"},{"id":"sudoku-e82cc12690c5","tier":"steady","k":1,"outcome":"no grid","correct":0,"blanks":53,"confidence":0,"tokens_out":2072,"cost":0.0028145400000000004,"finished_at":"2026-10-15T19:56:33+00:00"},{"id":"sudoku-76138868a3a5","tier":"gentle","k":1,"outcome":"changed givens","correct":15,"blanks":51,"confidence":0,"tokens_out":1205,"cost":0.00139284,"finished_at":"2026-10-15T19:54:33+00:00"},{"id":"sudoku-d4887e180643","tier":"cozy","k":1,"outcome":"incomplete","correct":0,"blanks":48,"confidence":0,"tokens_out":2854,"cost":0.0029399,"finished_at":"2026-10-15T19:51:33+00:00"},{"id":"sudoku-6de1c62e1201","tier":"beyond","k":0,"outcome":"changed givens","correct":14,"blanks":60,"confidence":0,"tokens_out":2184,"cost":0.00235682,"finished_at":"2026-10-15T19:49:25+00:00"},{"id":"sudoku-62755c9d0d0e","tier":"expert","k":0,"outcome":"no grid","correct":0,"blanks":58,"confidence":0,"tokens_out":3434,"cost":0.00508446,"finished_at":"2026-10-15T19:43:59+00:00"},{"id":"sudoku-46eddedc0ba0","tier":"devious","k":0,"outcome":"changed givens","correct":0,"blanks":55,"confidence":0,"tokens_out":1848,"cost":0.00294764,"finished_at":"2026-10-15T19:37:33+00:00"},{"id":"sudoku-4fa163656f33","tier":"tricky","k":0,"outcome":"changed givens","correct":0,"blanks":55,"confidence":0,"tokens_out":1120,"cost":0.0019217000000000001,"finished_at":"2026-10-15T19:35:06+00:00"},{"id":"sudoku-b67ffb3f4bad","tier":"steady","k":0,"outcome":"changed givens","correct":0,"blanks":53,"confidence":0,"tokens_out":2356,"cost":0.00360978,"finished_at":"2026-10-15T19:31:15+00:00"},{"id":"sudoku-d1eb2ba828b7","tier":"gentle","k":0,"outcome":"changed givens","correct":12,"blanks":56,"confidence":0,"tokens_out":1270,"cost":0.00202606,"finished_at":"2026-10-15T19:28:50+00:00"},{"id":"sudoku-df55b9f602dd","tier":"cozy","k":0,"outcome":"incomplete","correct":0,"blanks":50,"confidence":0,"tokens_out":1615,"cost":0.00226194,"finished_at":"2026-10-15T19:27:32+00:00"}]},{"id":"codex-gpt-6-sol-medium","model_id":"openai/gpt-6-sol","name":"GPT-6 Sol","lab":"openai","provider":"codex","route":"gpt-6-sol","effort":"medium","released":"2026-09-22","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":38,"solve_rate":0.905,"score":90.5,"score_low":80.5,"score_high":100.0,"tiers":{"cozy":{"n":6,"solved":6,"rate":1.0,"tokens_out":4229,"seconds":50.3},"gentle":{"n":6,"solved":6,"rate":1.0,"tokens_out":9606,"seconds":104.2},"steady":{"n":6,"solved":6,"rate":1.0,"tokens_out":11675,"seconds":124.1},"tricky":{"n":6,"solved":6,"rate":1.0,"tokens_out":15702,"seconds":172.4},"devious":{"n":6,"solved":4,"rate":0.667,"tokens_out":7839,"seconds":88.1},"expert":{"n":6,"solved":6,"rate":1.0,"tokens_out":12594,"seconds":136.4},"beyond":{"n":6,"solved":4,"rate":0.667,"tokens_out":28828,"seconds":316.6}},"cell_accuracy":0.9373,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.9977,"cells":432},"2":{"name":"Scanning","accuracy":0.9911,"cells":674},"3":{"name":"Locked candidates","accuracy":0.9453,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":1.0,"cells":258},"5":{"name":"Wings and fish","accuracy":0.7515,"cells":165},"6":{"name":"Deep wings and fish","accuracy":1.0,"cells":181},"7":{"name":"Search","accuracy":0.6908,"cells":249}},"outcomes":{"solved":38,"wrong":2,"incomplete":2,"changed givens":0,"no grid":0},"rule_breaking":2,"near_misses":0,"confidence_given":1.0,"confidence_mean":95.2,"confidence_when_unsolved":50.0,"confident_and_wrong":2,"brier":0.0476,"retries":1,"truncated_rate":0.0,"cost_known":true,"cost_total":6.2555,"cost_per_puzzle":0.1489,"cost_per_solve":0.1646,"tokens_in_per_puzzle":30983,"tokens_out_per_puzzle":12925,"tokens_reasoning_per_puzzle":12465,"seconds_per_puzzle":141.7,"last_attempt":"2026-10-15T21:50:34+00:00","attempt_list":[{"id":"sudoku-69011ba42154","tier":"beyond","k":5,"outcome":"incomplete","correct":0,"blanks":57,"confidence":0,"tokens_out":70219,"cost":0.7470356,"finished_at":"2026-10-15T21:50:34+00:00"},{"id":"sudoku-512883e1c861","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":10705,"cost":0.1261312,"finished_at":"2026-10-15T21:37:37+00:00"},{"id":"sudoku-1d7cc3da4742","tier":"devious","k":5,"outcome":"wrong","correct":33,"blanks":54,"confidence":100,"tokens_out":9159,"cost":0.1149172,"finished_at":"2026-10-15T21:34:46+00:00"},{"id":"sudoku-be319bbfa630","tier":"tricky","k":5,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":25999,"cost":0.31470240000000005,"finished_at":"2026-10-15T21:33:12+00:00"},{"id":"sudoku-9450b0f5c6dd","tier":"steady","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":11008,"cost":0.1284228,"finished_at":"2026-10-15T21:25:54+00:00"},{"id":"sudoku-d4f693275b81","tier":"gentle","k":5,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":7171,"cost":0.0789052,"finished_at":"2026-10-15T21:23:46+00:00"},{"id":"sudoku-9aba6db035f0","tier":"beyond","k":4,"outcome":"incomplete","correct":17,"blanks":59,"confidence":0,"tokens_out":17776,"cost":0.1960828,"finished_at":"2026-10-15T21:22:31+00:00"},{"id":"sudoku-c2c07a0eda20","tier":"cozy","k":5,"outcome":"solved","correct":39,"blanks":39,"confidence":100,"tokens_out":3109,"cost":0.0382512,"finished_at":"2026-10-15T21:20:55+00:00"},{"id":"sudoku-00ba26c3f743","tier":"expert","k":4,"outcome":"solved","correct":58,"blanks":58,"confidence":100,"tokens_out":11886,"cost":0.1261532,"finished_at":"2026-10-15T21:15:43+00:00"},{"id":"sudoku-d419e53f63bd","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":100,"tokens_out":5556,"cost":0.06284719999999999,"finished_at":"2026-10-15T21:12:18+00:00"},{"id":"sudoku-d74a50849d7f","tier":"tricky","k":4,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":9873,"cost":0.10859719999999999,"finished_at":"2026-10-15T21:10:12+00:00"},{"id":"sudoku-070a77b50c5e","tier":"steady","k":4,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":13807,"cost":0.1673732,"finished_at":"2026-10-15T21:07:30+00:00"},{"id":"sudoku-bf55e48ae73f","tier":"gentle","k":4,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":11127,"cost":0.1183732,"finished_at":"2026-10-15T21:04:48+00:00"},{"id":"sudoku-d56103b5e941","tier":"cozy","k":4,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":4261,"cost":0.0659772,"finished_at":"2026-10-15T21:02:10+00:00"},{"id":"sudoku-6fa50dd1a9fd","tier":"beyond","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":18065,"cost":0.2112592,"finished_at":"2026-10-15T20:58:10+00:00"},{"id":"sudoku-52a2ca4e92a3","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":9575,"cost":0.1056908,"finished_at":"2026-10-15T20:54:39+00:00"},{"id":"sudoku-78791341bcb1","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":12299,"cost":0.1506276,"finished_at":"2026-10-15T20:51:49+00:00"},{"id":"sudoku-9d2e80b941b8","tier":"devious","k":3,"outcome":"solved","correct":40,"blanks":40,"confidence":100,"tokens_out":4302,"cost":0.051794,"finished_at":"2026-10-15T20:51:49+00:00"},{"id":"sudoku-adccc52ff916","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":23165,"cost":0.2687752,"finished_at":"2026-10-15T20:46:52+00:00"},{"id":"sudoku-dcc32d8bb233","tier":"gentle","k":3,"outcome":"solved","correct":51,"blanks":51,"confidence":100,"tokens_out":7232,"cost":0.0795212,"finished_at":"2026-10-15T20:42:11+00:00"},{"id":"sudoku-fc58567482d9","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":4319,"cost":0.0502732,"finished_at":"2026-10-15T20:39:49+00:00"},{"id":"sudoku-8351fa666eed","tier":"beyond","k":2,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":26854,"cost":0.3385992,"finished_at":"2026-10-15T20:37:19+00:00"},{"id":"sudoku-5a9b417c3bcc","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":8595,"cost":0.0932852,"finished_at":"2026-10-15T20:30:53+00:00"},{"id":"sudoku-178455917208","tier":"devious","k":2,"outcome":"wrong","correct":29,"blanks":49,"confidence":100,"tokens_out":7701,"cost":0.0841872,"finished_at":"2026-10-15T20:26:33+00:00"},{"id":"sudoku-039c7e478cd0","tier":"tricky","k":2,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":18684,"cost":0.2269868,"finished_at":"2026-10-15T20:25:39+00:00"},{"id":"sudoku-fb73d907c689","tier":"steady","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":6259,"cost":0.06983919999999999,"finished_at":"2026-10-15T20:20:11+00:00"},{"id":"sudoku-b1e2835349b2","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":15272,"cost":0.1800632,"finished_at":"2026-10-15T20:17:42+00:00"},{"id":"sudoku-91b3b8203bb4","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":4952,"cost":0.0727392,"finished_at":"2026-10-15T20:12:13+00:00"},{"id":"sudoku-5c1a19c5b22f","tier":"beyond","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":16181,"cost":0.1743264,"finished_at":"2026-10-15T20:11:14+00:00"},{"id":"sudoku-1dcf37a9d45b","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":6360,"cost":0.088528,"finished_at":"2026-10-15T20:06:27+00:00"},{"id":"sudoku-3535bde1df9e","tier":"devious","k":1,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":11309,"cost":0.1264432,"finished_at":"2026-10-15T20:04:18+00:00"},{"id":"sudoku-f3d0e2039f2c","tier":"tricky","k":1,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":15483,"cost":0.20150520000000002,"finished_at":"2026-10-15T20:02:31+00:00"},{"id":"sudoku-3597767ef830","tier":"steady","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":7645,"cost":0.0835952,"finished_at":"2026-10-15T19:58:10+00:00"},{"id":"sudoku-301e3ab09277","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":100,"tokens_out":6949,"cost":0.0768952,"finished_at":"2026-10-15T19:55:41+00:00"},{"id":"sudoku-05bb142d4bfe","tier":"beyond","k":0,"outcome":"solved","correct":60,"blanks":60,"confidence":100,"tokens_out":23873,"cost":0.2541124,"finished_at":"2026-10-15T19:54:15+00:00"},{"id":"sudoku-cce0a72b1023","tier":"cozy","k":1,"outcome":"solved","correct":48,"blanks":48,"confidence":100,"tokens_out":4261,"cost":0.049819199999999994,"finished_at":"2026-10-15T19:52:27+00:00"},{"id":"sudoku-0966e5d017e5","tier":"expert","k":0,"outcome":"solved","correct":58,"blanks":58,"confidence":100,"tokens_out":28442,"cost":0.2994168,"finished_at":"2026-10-15T19:48:53+00:00"},{"id":"sudoku-e9520b4b4438","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":9006,"cost":0.09731519999999999,"finished_at":"2026-10-15T19:39:13+00:00"},{"id":"sudoku-2b304d23a7fd","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":11877,"cost":0.1450732,"finished_at":"2026-10-15T19:37:07+00:00"},{"id":"sudoku-e52c7a551fc9","tier":"steady","k":0,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":8168,"cost":0.0991832,"finished_at":"2026-10-15T19:32:49+00:00"},{"id":"sudoku-ed7587ae7df7","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":9882,"cost":0.12472319999999999,"finished_at":"2026-10-15T19:30:37+00:00"},{"id":"sudoku-2dfc55b6f481","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":4474,"cost":0.0571524,"finished_at":"2026-10-15T19:28:30+00:00"}]},{"id":"codex-gpt-6-1-sol-medium","model_id":"openai/gpt-6.1-sol","name":"GPT-6.1 Sol","lab":"openai","provider":"codex","route":"gpt-6.1-sol","effort":"medium","released":"2026-09-29","open_weights":false,"attempts":42,"expected":42,"aborted":0,"complete_set":true,"solved":35,"solve_rate":0.833,"score":83.3,"score_low":72.3,"score_high":94.4,"tiers":{"cozy":{"n":6,"solved":6,"rate":1.0,"tokens_out":3938,"seconds":94.9},"gentle":{"n":6,"solved":6,"rate":1.0,"tokens_out":6007,"seconds":135.6},"steady":{"n":6,"solved":4,"rate":0.667,"tokens_out":8432,"seconds":195.8},"tricky":{"n":6,"solved":5,"rate":0.833,"tokens_out":8794,"seconds":196.2},"devious":{"n":6,"solved":5,"rate":0.833,"tokens_out":6595,"seconds":153.3},"expert":{"n":6,"solved":4,"rate":0.667,"tokens_out":9598,"seconds":220.7},"beyond":{"n":6,"solved":5,"rate":0.833,"tokens_out":17701,"seconds":368.5}},"cell_accuracy":0.9409,"cells":2233,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":1.0,"cells":432},"2":{"name":"Scanning","accuracy":0.9985,"cells":674},"3":{"name":"Locked candidates","accuracy":0.9051,"cells":274},"4":{"name":"Hidden pairs and triples","accuracy":0.9109,"cells":258},"5":{"name":"Wings and fish","accuracy":0.9394,"cells":165},"6":{"name":"Deep wings and fish","accuracy":0.768,"cells":181},"7":{"name":"Search","accuracy":0.8795,"cells":249}},"outcomes":{"solved":35,"wrong":7,"incomplete":0,"changed givens":0,"no grid":0},"rule_breaking":7,"near_misses":0,"confidence_given":1.0,"confidence_mean":95.2,"confidence_when_unsolved":71.4,"confident_and_wrong":5,"brier":0.119,"retries":0,"truncated_rate":0.0,"cost_known":true,"cost_total":4.7013,"cost_per_puzzle":0.1119,"cost_per_solve":0.1343,"tokens_in_per_puzzle":27423,"tokens_out_per_puzzle":8723,"tokens_reasoning_per_puzzle":8345,"seconds_per_puzzle":195.0,"last_attempt":"2026-10-15T21:38:50+00:00","attempt_list":[{"id":"sudoku-4287e3c544a8","tier":"beyond","k":5,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":8666,"cost":0.134034,"finished_at":"2026-10-15T21:38:50+00:00"},{"id":"sudoku-874e275dd562","tier":"expert","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":5679,"cost":0.061805599999999995,"finished_at":"2026-10-15T21:35:35+00:00"},{"id":"sudoku-a54641a7f0b7","tier":"devious","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":4115,"cost":0.0632996,"finished_at":"2026-10-15T21:30:32+00:00"},{"id":"sudoku-288398e46763","tier":"tricky","k":5,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":10134,"cost":0.114246,"finished_at":"2026-10-15T21:28:51+00:00"},{"id":"sudoku-3ca4df83a251","tier":"steady","k":5,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":6975,"cost":0.0918616,"finished_at":"2026-10-15T21:25:12+00:00"},{"id":"sudoku-faec98cc7fbe","tier":"gentle","k":5,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":3167,"cost":0.0538396,"finished_at":"2026-10-15T21:22:01+00:00"},{"id":"sudoku-76b8a324102f","tier":"cozy","k":5,"outcome":"solved","correct":39,"blanks":39,"confidence":100,"tokens_out":1719,"cost":0.0279532,"finished_at":"2026-10-15T21:20:00+00:00"},{"id":"sudoku-0602ad60faaa","tier":"expert","k":4,"outcome":"wrong","correct":35,"blanks":58,"confidence":0,"tokens_out":19902,"cost":0.2482436,"finished_at":"2026-10-15T21:19:57+00:00"},{"id":"sudoku-d705d33efbb3","tier":"beyond","k":4,"outcome":"solved","correct":59,"blanks":59,"confidence":100,"tokens_out":9025,"cost":0.1009012,"finished_at":"2026-10-15T21:18:57+00:00"},{"id":"sudoku-0bad41db7466","tier":"devious","k":4,"outcome":"solved","correct":47,"blanks":47,"confidence":100,"tokens_out":6988,"cost":0.0797096,"finished_at":"2026-10-15T21:12:55+00:00"},{"id":"sudoku-040460fec51c","tier":"tricky","k":4,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":4780,"cost":0.07485560000000001,"finished_at":"2026-10-15T21:07:54+00:00"},{"id":"sudoku-4397ec455fc0","tier":"steady","k":4,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":4025,"cost":0.0487816,"finished_at":"2026-10-15T21:06:04+00:00"},{"id":"sudoku-ebe458443dde","tier":"gentle","k":4,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":9542,"cost":0.1175876,"finished_at":"2026-10-15T21:04:36+00:00"},{"id":"sudoku-ad5a5b9f6e3e","tier":"beyond","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":17190,"cost":0.2422536,"finished_at":"2026-10-15T21:00:53+00:00"},{"id":"sudoku-8a5f12917970","tier":"cozy","k":4,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":5912,"cost":0.08887160000000001,"finished_at":"2026-10-15T21:00:30+00:00"},{"id":"sudoku-c362fa7ad075","tier":"expert","k":3,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":8110,"cost":0.0861336,"finished_at":"2026-10-15T20:54:46+00:00"},{"id":"sudoku-9227bfc33d50","tier":"devious","k":3,"outcome":"solved","correct":40,"blanks":40,"confidence":100,"tokens_out":2672,"cost":0.037184,"finished_at":"2026-10-15T20:50:43+00:00"},{"id":"sudoku-abb4f3a03a0f","tier":"tricky","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":12103,"cost":0.14306560000000001,"finished_at":"2026-10-15T20:49:41+00:00"},{"id":"sudoku-41241c32f559","tier":"steady","k":3,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":10975,"cost":0.13809120000000003,"finished_at":"2026-10-15T20:45:32+00:00"},{"id":"sudoku-4b7be84c19da","tier":"gentle","k":3,"outcome":"solved","correct":51,"blanks":51,"confidence":100,"tokens_out":2728,"cost":0.040248,"finished_at":"2026-10-15T20:40:42+00:00"},{"id":"sudoku-e86fc0e760d2","tier":"cozy","k":3,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":9181,"cost":0.1263436,"finished_at":"2026-10-15T20:38:59+00:00"},{"id":"sudoku-4bd0d0f334e6","tier":"beyond","k":2,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":16873,"cost":0.1772976,"finished_at":"2026-10-15T20:35:17+00:00"},{"id":"sudoku-d64819170cd2","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":8647,"cost":0.1086096,"finished_at":"2026-10-15T20:28:54+00:00"},{"id":"sudoku-92cf78ad6753","tier":"devious","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":3912,"cost":0.08258,"finished_at":"2026-10-15T20:24:28+00:00"},{"id":"sudoku-9e01e97fb9ec","tier":"tricky","k":2,"outcome":"solved","correct":57,"blanks":57,"confidence":100,"tokens_out":12068,"cost":0.1421876,"finished_at":"2026-10-15T20:23:53+00:00"},{"id":"sudoku-b2c04fd41a5b","tier":"beyond","k":1,"outcome":"wrong","correct":27,"blanks":57,"confidence":0,"tokens_out":47554,"cost":0.5669856,"finished_at":"2026-10-15T20:21:51+00:00"},{"id":"sudoku-97cfb3efe08d","tier":"steady","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":8780,"cost":0.1099456,"finished_at":"2026-10-15T20:18:55+00:00"},{"id":"sudoku-8decd8f5b9e3","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":6051,"cost":0.0825936,"finished_at":"2026-10-15T20:14:41+00:00"},{"id":"sudoku-a42c15a683c8","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":2536,"cost":0.0474516,"finished_at":"2026-10-15T20:10:53+00:00"},{"id":"sudoku-87e7b09426ec","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":4874,"cost":0.0594672,"finished_at":"2026-10-15T20:04:36+00:00"},{"id":"sudoku-d1b9849f0b7d","tier":"devious","k":1,"outcome":"wrong","correct":45,"blanks":55,"confidence":100,"tokens_out":3459,"cost":0.047384,"finished_at":"2026-10-15T20:01:45+00:00"},{"id":"sudoku-a3f0e90cdffe","tier":"tricky","k":1,"outcome":"wrong","correct":34,"blanks":57,"confidence":100,"tokens_out":6585,"cost":0.1013612,"finished_at":"2026-10-15T20:00:16+00:00"},{"id":"sudoku-60e80c008543","tier":"steady","k":1,"outcome":"wrong","correct":41,"blanks":53,"confidence":100,"tokens_out":9115,"cost":0.1092376,"finished_at":"2026-10-15T19:58:51+00:00"},{"id":"sudoku-2f21d3120ee8","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":100,"tokens_out":9006,"cost":0.1263496,"finished_at":"2026-10-15T19:56:03+00:00"},{"id":"sudoku-bfa44d4d04a0","tier":"cozy","k":1,"outcome":"solved","correct":48,"blanks":48,"confidence":100,"tokens_out":1872,"cost":0.0410736,"finished_at":"2026-10-15T19:50:56+00:00"},{"id":"sudoku-195c6858bb96","tier":"beyond","k":0,"outcome":"solved","correct":60,"blanks":60,"confidence":100,"tokens_out":6897,"cost":0.1117936,"finished_at":"2026-10-15T19:46:37+00:00"},{"id":"sudoku-a438d1c276e4","tier":"expert","k":0,"outcome":"wrong","correct":39,"blanks":58,"confidence":100,"tokens_out":10374,"cost":0.12588960000000002,"finished_at":"2026-10-15T19:43:12+00:00"},{"id":"sudoku-2b2e2a133879","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":18426,"cost":0.2111536,"finished_at":"2026-10-15T19:42:05+00:00"},{"id":"sudoku-06181f096bb3","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":7097,"cost":0.0816192,"finished_at":"2026-10-15T19:34:56+00:00"},{"id":"sudoku-1c6de7197f72","tier":"steady","k":0,"outcome":"wrong","correct":38,"blanks":53,"confidence":100,"tokens_out":10720,"cost":0.12930160000000002,"finished_at":"2026-10-15T19:34:48+00:00"},{"id":"sudoku-3ae875284721","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":5547,"cost":0.08492960000000001,"finished_at":"2026-10-15T19:30:09+00:00"},{"id":"sudoku-83736b25b886","tier":"cozy","k":0,"outcome":"solved","correct":50,"blanks":50,"confidence":100,"tokens_out":2405,"cost":0.0347452,"finished_at":"2026-10-15T19:28:09+00:00"}]},{"id":"local-gemma-3-4b-none","model_id":"google/gemma-3-4b","name":"Gemma 3 4B","lab":"google","provider":"ollama","route":"gemma3:4b","effort":"none","released":"2025-03-12","open_weights":true,"attempts":7,"expected":42,"aborted":0,"complete_set":false,"solved":0,"solve_rate":0.0,"score":0.0,"score_low":0.0,"score_high":20.2,"tiers":{"cozy":{"n":1,"solved":0,"rate":0.0,"tokens_out":142,"seconds":16.2},"gentle":{"n":1,"solved":0,"rate":0.0,"tokens_out":295,"seconds":15.3},"steady":{"n":1,"solved":0,"rate":0.0,"tokens_out":133,"seconds":14.2},"tricky":{"n":1,"solved":0,"rate":0.0,"tokens_out":259,"seconds":36.9},"devious":{"n":1,"solved":0,"rate":0.0,"tokens_out":132,"seconds":6.6},"expert":{"n":1,"solved":0,"rate":0.0,"tokens_out":149,"seconds":24.2},"beyond":{"n":1,"solved":0,"rate":0.0,"tokens_out":127,"seconds":14.2}},"cell_accuracy":0.0672,"cells":387,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.1148,"cells":61},"2":{"name":"Scanning","accuracy":0.1013,"cells":79},"3":{"name":"Locked candidates","accuracy":0.0,"cells":46},"4":{"name":"Hidden pairs and triples","accuracy":1.0,"cells":1},"5":{"name":"Wings and fish","accuracy":0.0909,"cells":22},"6":{"name":"Deep wings and fish","accuracy":0.1471,"cells":34},"7":{"name":"Search","accuracy":0.0909,"cells":33}},"outcomes":{"solved":0,"wrong":0,"incomplete":0,"changed givens":5,"no grid":2},"rule_breaking":0,"near_misses":0,"confidence_given":1.0,"confidence_mean":96.1,"confidence_when_unsolved":96.1,"confident_and_wrong":7,"brier":0.9247,"retries":2,"truncated_rate":0.0,"cost_known":false,"cost_total":null,"cost_per_puzzle":null,"cost_per_solve":null,"tokens_in_per_puzzle":464,"tokens_out_per_puzzle":177,"tokens_reasoning_per_puzzle":0,"seconds_per_puzzle":18.2,"last_attempt":"2026-10-19T19:52:27+00:00","attempt_list":[{"id":"sudoku-de2f43d3f037","tier":"beyond","k":0,"outcome":"changed givens","correct":7,"blanks":60,"confidence":95,"tokens_out":127,"cost":0.0,"finished_at":"2026-10-19T19:52:27+00:00"},{"id":"sudoku-c2eb16a76587","tier":"expert","k":0,"outcome":"changed givens","correct":7,"blanks":58,"confidence":95,"tokens_out":149,"cost":0.0,"finished_at":"2026-10-19T15:51:38+00:00"},{"id":"sudoku-787860049cb7","tier":"devious","k":0,"outcome":"changed givens","correct":6,"blanks":55,"confidence":95,"tokens_out":132,"cost":0.0,"finished_at":"2026-10-19T12:46:20+00:00"},{"id":"sudoku-c122d1db963d","tier":"tricky","k":0,"outcome":"no grid","correct":0,"blanks":55,"confidence":99,"tokens_out":259,"cost":0.0,"finished_at":"2026-10-17T03:33:02+00:00"},{"id":"sudoku-0c4d9bb50e7c","tier":"steady","k":0,"outcome":"changed givens","correct":2,"blanks":53,"confidence":95,"tokens_out":133,"cost":0.0,"finished_at":"2026-10-16T14:29:47+00:00"},{"id":"sudoku-facae8d2fb58","tier":"gentle","k":0,"outcome":"no grid","correct":0,"blanks":56,"confidence":99,"tokens_out":295,"cost":0.0,"finished_at":"2026-10-16T14:04:00+00:00"},{"id":"sudoku-d667ddded0ae","tier":"cozy","k":0,"outcome":"changed givens","correct":4,"blanks":50,"confidence":95,"tokens_out":142,"cost":0.0,"finished_at":"2026-10-16T13:31:40+00:00"}]},{"id":"local-ministral-3-8b-none","model_id":"mistral/ministral-3-8b","name":"Ministral 3 8B","lab":"mistral","provider":"ollama","route":"ministral-3:8b-instruct-2512-q4_K_M","effort":"none","released":"2025-12-02","open_weights":true,"attempts":7,"expected":42,"aborted":0,"complete_set":false,"solved":0,"solve_rate":0.0,"score":0.0,"score_low":0.0,"score_high":20.2,"tiers":{"cozy":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":20.5},"gentle":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":22.4},"steady":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":22.7},"tricky":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":20.4},"devious":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":23.9},"expert":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":11.2},"beyond":{"n":1,"solved":0,"rate":0.0,"tokens_out":108,"seconds":16.1}},"cell_accuracy":0.124,"cells":387,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.0968,"cells":62},"2":{"name":"Scanning","accuracy":0.1387,"cells":137},"3":{"name":"Locked candidates","accuracy":0.087,"cells":46},"4":{"name":"Hidden pairs and triples","accuracy":0.1132,"cells":53},"5":{"name":"Wings and fish","accuracy":0.0909,"cells":22},"6":{"name":"Deep wings and fish","accuracy":0.2059,"cells":34},"7":{"name":"Search","accuracy":0.1212,"cells":33}},"outcomes":{"solved":0,"wrong":0,"incomplete":0,"changed givens":7,"no grid":0},"rule_breaking":0,"near_misses":0,"confidence_given":1.0,"confidence_mean":100.0,"confidence_when_unsolved":100.0,"confident_and_wrong":7,"brier":1.0,"retries":0,"truncated_rate":0.0,"cost_known":false,"cost_total":null,"cost_per_puzzle":null,"cost_per_solve":null,"tokens_in_per_puzzle":304,"tokens_out_per_puzzle":108,"tokens_reasoning_per_puzzle":0,"seconds_per_puzzle":19.6,"last_attempt":"2026-10-24T06:19:15+00:00","attempt_list":[{"id":"sudoku-86b07c08731e","tier":"beyond","k":0,"outcome":"changed givens","correct":11,"blanks":60,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-24T06:19:15+00:00"},{"id":"sudoku-a1fc2978ec85","tier":"expert","k":0,"outcome":"changed givens","correct":11,"blanks":58,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-19T19:25:58+00:00"},{"id":"sudoku-1dec5298ab0c","tier":"devious","k":0,"outcome":"changed givens","correct":3,"blanks":55,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-19T15:21:26+00:00"},{"id":"sudoku-01dba8866051","tier":"tricky","k":0,"outcome":"changed givens","correct":6,"blanks":55,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-18T14:37:53+00:00"},{"id":"sudoku-27aa0f1ce65c","tier":"steady","k":0,"outcome":"changed givens","correct":5,"blanks":53,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-17T03:13:17+00:00"},{"id":"sudoku-db59262bd7b3","tier":"gentle","k":0,"outcome":"changed givens","correct":7,"blanks":56,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-16T14:15:16+00:00"},{"id":"sudoku-9c28dc72fc66","tier":"cozy","k":0,"outcome":"changed givens","correct":5,"blanks":50,"confidence":100,"tokens_out":108,"cost":0.0,"finished_at":"2026-10-16T13:44:36+00:00"}]},{"id":"local-qwen3-5-9b-none","model_id":"alibaba/qwen3.5-9b","name":"Qwen3.5 9B","lab":"alibaba","provider":"ollama","route":"qwen3.5:9b","effort":"none","released":"2026-02-23","open_weights":true,"attempts":7,"expected":42,"aborted":0,"complete_set":false,"solved":0,"solve_rate":0.0,"score":0.0,"score_low":0.0,"score_high":20.2,"tiers":{"cozy":{"n":1,"solved":0,"rate":0.0,"tokens_out":3000,"seconds":342.5},"gentle":{"n":1,"solved":0,"rate":0.0,"tokens_out":3000,"seconds":357.4},"steady":{"n":1,"solved":0,"rate":0.0,"tokens_out":3000,"seconds":317.6},"tricky":{"n":1,"solved":0,"rate":0.0,"tokens_out":3000,"seconds":278.1},"devious":{"n":1,"solved":0,"rate":0.0,"tokens_out":1606,"seconds":205.6},"expert":{"n":1,"solved":0,"rate":0.0,"tokens_out":1605,"seconds":66.3},"beyond":{"n":1,"solved":0,"rate":0.0,"tokens_out":1605,"seconds":122.6}},"cell_accuracy":0.093,"cells":387,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.25,"cells":4},"2":{"name":"Scanning","accuracy":0.1463,"cells":123},"3":{"name":"Locked candidates","accuracy":0.0833,"cells":12},"4":{"name":"Hidden pairs and triples","accuracy":0.0,"cells":1},"5":{"name":"Wings and fish","accuracy":0.2273,"cells":22},"6":{"name":"Deep wings and fish","accuracy":0.2941,"cells":34},"7":{"name":"Search","accuracy":0.0303,"cells":33}},"outcomes":{"solved":0,"wrong":0,"incomplete":0,"changed givens":4,"no grid":3},"rule_breaking":0,"near_misses":0,"confidence_given":0.4286,"confidence_mean":0.0,"confidence_when_unsolved":0.0,"confident_and_wrong":0,"brier":0.0,"retries":7,"truncated_rate":0.7857,"cost_known":false,"cost_total":null,"cost_per_puzzle":null,"cost_per_solve":null,"tokens_in_per_puzzle":1732,"tokens_out_per_puzzle":2402,"tokens_reasoning_per_puzzle":0,"seconds_per_puzzle":241.5,"last_attempt":"2026-10-24T06:21:19+00:00","attempt_list":[{"id":"sudoku-a4824b06f6c3","tier":"beyond","k":0,"outcome":"changed givens","correct":6,"blanks":60,"confidence":0,"tokens_out":1605,"cost":0.0,"finished_at":"2026-10-24T06:21:19+00:00"},{"id":"sudoku-625c5bc1386c","tier":"expert","k":0,"outcome":"changed givens","correct":13,"blanks":58,"confidence":0,"tokens_out":1605,"cost":0.0,"finished_at":"2026-10-19T19:27:06+00:00"},{"id":"sudoku-b2dc70e5825d","tier":"devious","k":0,"outcome":"changed givens","correct":10,"blanks":55,"confidence":0,"tokens_out":1606,"cost":0.0,"finished_at":"2026-10-19T15:24:52+00:00"},{"id":"sudoku-edd53f6d3803","tier":"tricky","k":0,"outcome":"no grid","correct":0,"blanks":55,"confidence":null,"tokens_out":3000,"cost":0.0,"finished_at":"2026-10-18T14:42:32+00:00"},{"id":"sudoku-96d8878bda3f","tier":"steady","k":0,"outcome":"no grid","correct":0,"blanks":53,"confidence":null,"tokens_out":3000,"cost":0.0,"finished_at":"2026-10-17T03:18:36+00:00"},{"id":"sudoku-95189eca2f30","tier":"gentle","k":0,"outcome":"changed givens","correct":7,"blanks":56,"confidence":null,"tokens_out":3000,"cost":0.0,"finished_at":"2026-10-16T14:21:14+00:00"},{"id":"sudoku-eac531e4eca1","tier":"cozy","k":0,"outcome":"no grid","correct":0,"blanks":50,"confidence":null,"tokens_out":3000,"cost":0.0,"finished_at":"2026-10-16T13:50:18+00:00"}]},{"id":"local-qwen3-8-27b-none","model_id":"alibaba/qwen3.8-27b","name":"Qwen3.8 27B","lab":"alibaba","provider":"ollama","route":"qwen3.8:27b","effort":"none","released":"2026-08-14","open_weights":true,"attempts":7,"expected":42,"aborted":0,"complete_set":false,"solved":0,"solve_rate":0.0,"score":0.0,"score_low":0.0,"score_high":20.2,"tiers":{"cozy":{"n":1,"solved":0,"rate":0.0,"tokens_out":18402,"seconds":804.4},"gentle":{"n":1,"solved":0,"rate":0.0,"tokens_out":10077,"seconds":497.4},"steady":{"n":1,"solved":0,"rate":0.0,"tokens_out":11476,"seconds":827.7},"tricky":{"n":1,"solved":0,"rate":0.0,"tokens_out":11251,"seconds":3239.5},"devious":{"n":1,"solved":0,"rate":0.0,"tokens_out":10583,"seconds":1579.5},"expert":{"n":1,"solved":0,"rate":0.0,"tokens_out":9257,"seconds":1505.3},"beyond":{"n":1,"solved":0,"rate":0.0,"tokens_out":15116,"seconds":819.5}},"cell_accuracy":0.2946,"cells":387,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.5,"cells":62},"2":{"name":"Scanning","accuracy":0.3212,"cells":137},"3":{"name":"Locked candidates","accuracy":0.2391,"cells":46},"4":{"name":"Hidden pairs and triples","accuracy":0.1698,"cells":53},"5":{"name":"Wings and fish","accuracy":0.3636,"cells":22},"6":{"name":"Deep wings and fish","accuracy":0.0882,"cells":34},"7":{"name":"Search","accuracy":0.2424,"cells":33}},"outcomes":{"solved":0,"wrong":2,"incomplete":0,"changed givens":5,"no grid":0},"rule_breaking":2,"near_misses":0,"confidence_given":1.0,"confidence_mean":71.4,"confidence_when_unsolved":71.4,"confident_and_wrong":4,"brier":0.5929,"retries":2,"truncated_rate":0.0,"cost_known":false,"cost_total":null,"cost_per_puzzle":null,"cost_per_solve":null,"tokens_in_per_puzzle":726,"tokens_out_per_puzzle":12309,"tokens_reasoning_per_puzzle":0,"seconds_per_puzzle":1324.8,"last_attempt":"2026-10-24T06:34:59+00:00","attempt_list":[{"id":"sudoku-05dbe3f3445a","tier":"beyond","k":0,"outcome":"wrong","correct":20,"blanks":60,"confidence":50,"tokens_out":15116,"cost":0.0,"finished_at":"2026-10-24T06:34:59+00:00"},{"id":"sudoku-0ce382cf2cc2","tier":"expert","k":0,"outcome":"changed givens","correct":10,"blanks":58,"confidence":20,"tokens_out":9257,"cost":0.0,"finished_at":"2026-10-19T19:52:11+00:00"},{"id":"sudoku-572208fb7528","tier":"devious","k":0,"outcome":"changed givens","correct":15,"blanks":55,"confidence":95,"tokens_out":10583,"cost":0.0,"finished_at":"2026-10-19T15:51:12+00:00"},{"id":"sudoku-6aeb5bf26661","tier":"tricky","k":0,"outcome":"changed givens","correct":9,"blanks":55,"confidence":50,"tokens_out":11251,"cost":0.0,"finished_at":"2026-10-19T12:46:12+00:00"},{"id":"sudoku-84e27db5f39f","tier":"steady","k":0,"outcome":"wrong","correct":23,"blanks":53,"confidence":95,"tokens_out":11476,"cost":0.0,"finished_at":"2026-10-17T03:32:24+00:00"},{"id":"sudoku-115c28315bc5","tier":"gentle","k":0,"outcome":"changed givens","correct":12,"blanks":56,"confidence":95,"tokens_out":10077,"cost":0.0,"finished_at":"2026-10-16T14:29:31+00:00"},{"id":"sudoku-7731507dcf86","tier":"cozy","k":0,"outcome":"changed givens","correct":25,"blanks":50,"confidence":95,"tokens_out":18402,"cost":0.0,"finished_at":"2026-10-16T14:03:43+00:00"}]},{"id":"local-qwen3-8-flash-next-low","model_id":"alibaba/qwen3.8-flash-next","name":"Qwen3.8 Flash-Next","lab":"alibaba","provider":"openai-compat","route":"qwen3.8-flash-next","effort":"low","released":"2026-08-26","open_weights":true,"attempts":21,"expected":42,"aborted":0,"complete_set":false,"solved":11,"solve_rate":0.524,"score":52.4,"score_low":36.9,"score_high":67.8,"tiers":{"cozy":{"n":3,"solved":2,"rate":0.667,"tokens_out":37321,"seconds":1760.3},"gentle":{"n":3,"solved":3,"rate":1.0,"tokens_out":36521,"seconds":1761.7},"steady":{"n":3,"solved":1,"rate":0.333,"tokens_out":91260,"seconds":4620.9},"tricky":{"n":3,"solved":1,"rate":0.333,"tokens_out":72341,"seconds":4931.6},"devious":{"n":3,"solved":2,"rate":0.667,"tokens_out":54011,"seconds":2694.2},"expert":{"n":3,"solved":2,"rate":0.667,"tokens_out":64097,"seconds":3239.6},"beyond":{"n":3,"solved":0,"rate":0.0,"tokens_out":94361,"seconds":4892.8}},"cell_accuracy":0.5102,"cells":1127,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":1.0,"cells":131},"2":{"name":"Scanning","accuracy":1.0,"cells":256},"3":{"name":"Locked candidates","accuracy":1.0,"cells":42},"4":{"name":"Hidden pairs and triples","accuracy":1.0,"cells":52},"5":{"name":"Wings and fish","accuracy":1.0,"cells":48},"6":{"name":"Deep wings and fish","accuracy":1.0,"cells":46}},"outcomes":{"solved":11,"wrong":0,"incomplete":0,"changed givens":0,"no grid":10},"rule_breaking":0,"near_misses":0,"confidence_given":0.5238,"confidence_mean":100.0,"confidence_when_unsolved":null,"confident_and_wrong":0,"brier":0.0,"retries":13,"truncated_rate":0.0,"cost_known":false,"cost_total":null,"cost_per_puzzle":null,"cost_per_solve":null,"tokens_in_per_puzzle":605,"tokens_out_per_puzzle":64273,"tokens_reasoning_per_puzzle":0,"seconds_per_puzzle":3414.4,"last_attempt":"2026-10-25T16:24:37+00:00","attempt_list":[{"id":"sudoku-a2d57f864ec3","tier":"beyond","k":2,"outcome":"no grid","correct":0,"blanks":56,"confidence":null,"tokens_out":94025,"cost":0.0,"finished_at":"2026-10-25T16:24:37+00:00"},{"id":"sudoku-ed54478bcffe","tier":"expert","k":2,"outcome":"solved","correct":54,"blanks":54,"confidence":100,"tokens_out":41640,"cost":0.0,"finished_at":"2026-10-25T14:58:42+00:00"},{"id":"sudoku-5a3434498d20","tier":"devious","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":27992,"cost":0.0,"finished_at":"2026-10-25T14:22:55+00:00"},{"id":"sudoku-6e14d949ea83","tier":"tricky","k":2,"outcome":"no grid","correct":0,"blanks":57,"confidence":null,"tokens_out":91670,"cost":0.0,"finished_at":"2026-10-24T03:45:42+00:00"},{"id":"sudoku-584edc44638c","tier":"steady","k":2,"outcome":"no grid","correct":0,"blanks":49,"confidence":null,"tokens_out":84106,"cost":0.0,"finished_at":"2026-10-24T01:20:40+00:00"},{"id":"sudoku-d9900d1c5d85","tier":"gentle","k":2,"outcome":"solved","correct":52,"blanks":52,"confidence":100,"tokens_out":23580,"cost":0.0,"finished_at":"2026-10-24T00:07:11+00:00"},{"id":"sudoku-190677014e12","tier":"cozy","k":2,"outcome":"solved","correct":49,"blanks":49,"confidence":100,"tokens_out":13547,"cost":0.0,"finished_at":"2026-10-23T23:48:48+00:00"},{"id":"sudoku-f1da04453026","tier":"beyond","k":1,"outcome":"no grid","correct":0,"blanks":57,"confidence":null,"tokens_out":87683,"cost":0.0,"finished_at":"2026-10-23T09:49:43+00:00"},{"id":"sudoku-250a10f3d35f","tier":"expert","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":58312,"cost":0.0,"finished_at":"2026-10-23T04:40:21+00:00"},{"id":"sudoku-201a8c69d090","tier":"devious","k":1,"outcome":"no grid","correct":0,"blanks":55,"confidence":null,"tokens_out":98063,"cost":0.0,"finished_at":"2026-10-23T03:51:05+00:00"},{"id":"sudoku-a5041c6ffa49","tier":"tricky","k":1,"outcome":"no grid","correct":0,"blanks":57,"confidence":null,"tokens_out":103213,"cost":0.0,"finished_at":"2026-10-23T02:26:17+00:00"},{"id":"sudoku-b2a0f29f605f","tier":"steady","k":1,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":89458,"cost":0.0,"finished_at":"2026-10-23T01:00:18+00:00"},{"id":"sudoku-993bcfc0401a","tier":"gentle","k":1,"outcome":"solved","correct":51,"blanks":51,"confidence":100,"tokens_out":15630,"cost":0.0,"finished_at":"2026-10-22T23:47:18+00:00"},{"id":"sudoku-3b289918984e","tier":"cozy","k":1,"outcome":"solved","correct":48,"blanks":48,"confidence":100,"tokens_out":14860,"cost":0.0,"finished_at":"2026-10-22T23:35:00+00:00"},{"id":"sudoku-b0e127b78338","tier":"beyond","k":0,"outcome":"no grid","correct":0,"blanks":60,"confidence":null,"tokens_out":101374,"cost":0.0,"finished_at":"2026-10-22T20:53:39+00:00"},{"id":"sudoku-d4a61197207d","tier":"expert","k":0,"outcome":"no grid","correct":0,"blanks":58,"confidence":null,"tokens_out":92340,"cost":0.0,"finished_at":"2026-10-22T19:28:28+00:00"},{"id":"sudoku-1daabb15c2a9","tier":"devious","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":35979,"cost":0.0,"finished_at":"2026-10-22T18:11:28+00:00"},{"id":"sudoku-20e3e94adc9c","tier":"tricky","k":0,"outcome":"solved","correct":55,"blanks":55,"confidence":100,"tokens_out":22139,"cost":0.0,"finished_at":"2026-10-22T17:43:00+00:00"},{"id":"sudoku-c49e71b4a670","tier":"steady","k":0,"outcome":"no grid","correct":0,"blanks":53,"confidence":null,"tokens_out":100216,"cost":0.0,"finished_at":"2026-10-22T17:09:23+00:00"},{"id":"sudoku-36208c0c1c1c","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":70352,"cost":0.0,"finished_at":"2026-10-22T15:44:44+00:00"},{"id":"sudoku-0bc6bcc541e1","tier":"cozy","k":0,"outcome":"no grid","correct":0,"blanks":50,"confidence":null,"tokens_out":83556,"cost":0.0,"finished_at":"2026-10-22T14:47:16+00:00"}]},{"id":"local-gpt-oss-20b-low","model_id":"openai/gpt-oss-20b","name":"gpt-oss 20B","lab":"openai","provider":"ollama","route":"gpt-oss:20b","effort":"low","released":"2025-08-05","open_weights":true,"attempts":5,"expected":42,"aborted":98,"complete_set":false,"solved":2,"solve_rate":0.4,"score":40.0,"score_low":16.1,"score_high":63.9,"tiers":{"cozy":{"n":1,"solved":0,"rate":0.0,"tokens_out":8212,"seconds":753.4},"gentle":{"n":1,"solved":1,"rate":1.0,"tokens_out":18046,"seconds":653.4},"steady":{"n":1,"solved":1,"rate":1.0,"tokens_out":16661,"seconds":3345.9},"tricky":{"n":1,"solved":0,"rate":0.0,"tokens_out":64841,"seconds":5312.7},"devious":{"n":1,"solved":0,"rate":0.0,"tokens_out":15941,"seconds":3445.3},"expert":{"n":0,"solved":0,"rate":null,"tokens_out":null,"seconds":null},"beyond":{"n":0,"solved":0,"rate":null,"tokens_out":null,"seconds":null}},"cell_accuracy":0.7472,"cells":269,"cell_accuracy_by_rank":{"1":{"name":"Last free cell","accuracy":0.9333,"cells":60},"2":{"name":"Scanning","accuracy":0.9691,"cells":97},"3":{"name":"Locked candidates","accuracy":1.0,"cells":37},"5":{"name":"Wings and fish","accuracy":0.7,"cells":20}},"outcomes":{"solved":2,"wrong":0,"incomplete":0,"changed givens":2,"no grid":1},"rule_breaking":0,"near_misses":0,"confidence_given":0.8,"confidence_mean":100.0,"confidence_when_unsolved":100.0,"confident_and_wrong":2,"brier":0.5,"retries":1,"truncated_rate":0.3333,"cost_known":false,"cost_total":null,"cost_per_puzzle":null,"cost_per_solve":null,"tokens_in_per_puzzle":412,"tokens_out_per_puzzle":24740,"tokens_reasoning_per_puzzle":0,"seconds_per_puzzle":2702.1,"last_attempt":"2026-10-23T21:08:15+00:00","attempt_list":[{"id":"sudoku-4f9332338de2","tier":"tricky","k":0,"outcome":"no grid","correct":0,"blanks":55,"confidence":null,"tokens_out":64841,"cost":0.0,"finished_at":"2026-10-23T21:08:15+00:00"},{"id":"sudoku-20e2dc3dd22b","tier":"devious","k":0,"outcome":"changed givens","correct":46,"blanks":55,"confidence":100,"tokens_out":15941,"cost":0.0,"finished_at":"2026-10-19T13:43:45+00:00"},{"id":"sudoku-21921f37a1d1","tier":"steady","k":0,"outcome":"solved","correct":53,"blanks":53,"confidence":100,"tokens_out":16661,"cost":0.0,"finished_at":"2026-10-16T15:25:32+00:00"},{"id":"sudoku-fb44b2a360f3","tier":"gentle","k":0,"outcome":"solved","correct":56,"blanks":56,"confidence":100,"tokens_out":18046,"cost":0.0,"finished_at":"2026-10-16T14:14:53+00:00"},{"id":"sudoku-1686366667b9","tier":"cozy","k":0,"outcome":"changed givens","correct":46,"blanks":50,"confidence":100,"tokens_out":8212,"cost":0.0,"finished_at":"2026-10-16T13:44:14+00:00"}]}],"queued":[],"totals":{"attempts":390,"cost":84.32,"aborted":98,"solved":249}}