{"schema_version": 1, "checked_at": "2026-10-08T08:27:45Z", "refresh_hours": 24, "suites": [{"id": "terminal-4", "name": "Terminal-Bench 4.0", "metric": "Task success", "description": "Agent and model runs on terminal tasks. Published accuracy, with 95% confidence intervals where reported.", "historical": false, "rows": [{"id": "terminal-4:61f5fa5816e420de", "model": "Opus 5.5", "agent": "Claude Code", "agent_version": "2.1.284", "setting": "max", "score": 64.85, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-22", "cost_usd": 4718.65, "cost_per_trial_usd": 14.298939393939392, "seconds": 4790.4, "uncertainty": 3.11, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-30-anthropic-claude-opus-5-5-max-claude-code.json", "cohort": "published-suite", "file": "2026-09-30-anthropic-claude-opus-5-5-max-claude-code.json"}, {"id": "terminal-4:dc9be9908290bd07", "model": "Sonnet 5.5", "agent": "Claude Code", "agent_version": "2.1.284", "setting": "max", "score": 61.82, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-28", "cost_usd": 7333.64, "cost_per_trial_usd": 22.223151515151518, "seconds": 7431.6, "uncertainty": 2.94, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-30-anthropic-claude-sonnet-5-5-max-claude-code.json", "cohort": "published-suite", "file": "2026-09-30-anthropic-claude-sonnet-5-5-max-claude-code.json"}, {"id": "terminal-4:f6de35652ac9f4c0", "model": "GPT-6.1 Sol", "agent": "Codex", "agent_version": "0.159.0", "setting": "max", "score": 58.18, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-29", "cost_usd": 634.24, "cost_per_trial_usd": 1.921939393939394, "seconds": 3176.7, "uncertainty": 3.11, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-30-openai-gpt-6-1-sol-max-codex.json", "cohort": "published-suite", "file": "2026-09-30-openai-gpt-6-1-sol-max-codex.json"}, {"id": "terminal-4:53750b794ead615b", "model": "Fable 5.1", "agent": "Claude Code", "agent_version": "2.1.257", "setting": "max", "score": 57.88, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-01", "cost_usd": 6243.5, "cost_per_trial_usd": 18.919696969696968, "seconds": 3893.5, "uncertainty": 3.76, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-02-anthropic-claude-fable-5-1-max-claude-code.json", "cohort": "published-suite", "file": "2026-09-02-anthropic-claude-fable-5-1-max-claude-code.json"}, {"id": "terminal-4:f43aef08cd191c21", "model": "Opus 5", "agent": "Claude Code", "agent_version": "2.1.231", "setting": "max", "score": 51.82, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-07-24", "cost_usd": 5969.11, "cost_per_trial_usd": 18.08821212121212, "seconds": 4792.8, "uncertainty": 3.39, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-anthropic-claude-opus-5-max-claude-code.json", "cohort": "published-suite", "file": "2026-08-26-anthropic-claude-opus-5-max-claude-code.json"}, {"id": "terminal-4:75db002c4d9b237d", "model": "GPT-6 Sol", "agent": "Codex", "agent_version": "0.159.0", "setting": "max", "score": 49.39, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-22", "cost_usd": 1451.32, "cost_per_trial_usd": 4.397939393939394, "seconds": 2662.2, "uncertainty": 3.23, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-30-openai-gpt-6-sol-max-codex.json", "cohort": "published-suite", "file": "2026-09-30-openai-gpt-6-sol-max-codex.json"}, {"id": "terminal-4:d30d609b92750e18", "model": "Fable 5", "agent": "Claude Code", "agent_version": "2.1.231", "setting": "max", "score": 44.55, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-06-09", "cost_usd": 7265.01, "cost_per_trial_usd": 22.01518181818182, "seconds": 4202.8, "uncertainty": 3.85, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-anthropic-claude-fable-5-max-claude-code.json", "cohort": "published-suite", "file": "2026-08-26-anthropic-claude-fable-5-max-claude-code.json"}, {"id": "terminal-4:f96d22b02fbf4f9c", "model": "GLM-5.3", "agent": "Claude Code", "agent_version": "2.1.207", "setting": "max", "score": 41.82, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-08-14", "cost_usd": 2727.63, "cost_per_trial_usd": 8.265545454545455, "seconds": 5831.0, "uncertainty": 3.23, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-anthropic-glm-5-3-max-claude-code.json", "cohort": "published-suite", "file": "2026-08-26-anthropic-glm-5-3-max-claude-code.json"}, {"id": "terminal-4:c6c980ef8d1092b3", "model": "GPT-5.6 Sol", "agent": "Codex", "agent_version": "0.149.1", "setting": "max", "score": 37.27, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-06-26", "cost_usd": 2541.7, "cost_per_trial_usd": 7.702121212121211, "seconds": 2388.4, "uncertainty": 3.78, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-openai-gpt-5-6-sol-max-codex.json", "cohort": "published-suite", "file": "2026-08-26-openai-gpt-5-6-sol-max-codex.json"}, {"id": "terminal-4:8bead6a3e27637c7", "model": "GLM-5.3-Flash", "agent": "Claude Code", "agent_version": "2.1.285", "setting": "none", "score": 35.76, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-08-26", "cost_usd": 2883.11, "cost_per_trial_usd": 8.73669696969697, "seconds": 6209.8, "uncertainty": 3.54, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-10-06-z-ai-glm-5-3-flash-none-claude-code.json", "cohort": "published-suite", "file": "2026-10-06-z-ai-glm-5-3-flash-none-claude-code.json"}, {"id": "terminal-4:cd0e04915872d51d", "model": "Qwen3.8-Max-0902", "agent": "Claude Code", "agent_version": "2.1.285", "setting": "max", "score": 26.97, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-02", "cost_usd": 4635.67, "cost_per_trial_usd": 14.04748484848485, "seconds": 9953.0, "uncertainty": 3.78, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-10-06-qwen-qwen3-8-max-0902-max-claude-code.json", "cohort": "published-suite", "file": "2026-10-06-qwen-qwen3-8-max-0902-max-claude-code.json"}, {"id": "terminal-4:1605c4ffc303d26d", "model": "Opus 4.8", "agent": "Claude Code", "agent_version": "2.1.231", "setting": "max", "score": 23.64, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-05-28", "cost_usd": 6481.26, "cost_per_trial_usd": 19.64018181818182, "seconds": 5163.0, "uncertainty": 3.56, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-anthropic-claude-opus-4-8-max-claude-code.json", "cohort": "published-suite", "file": "2026-08-26-anthropic-claude-opus-4-8-max-claude-code.json"}, {"id": "terminal-4:199b0aec9d862e3e", "model": "GPT-5.6 Terra", "agent": "Codex", "agent_version": "0.149.1", "setting": "max", "score": 21.52, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-06-26", "cost_usd": 1733.52, "cost_per_trial_usd": 5.2530909090909095, "seconds": 2514.6, "uncertainty": 3.25, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-openai-gpt-5-6-terra-max-codex.json", "cohort": "published-suite", "file": "2026-08-26-openai-gpt-5-6-terra-max-codex.json"}, {"id": "terminal-4:e2fbb4608d461d85", "model": "Grok 4.6", "agent": "Grok Build", "agent_version": "1.0.5", "setting": "none", "score": 20.3, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-08-12", "cost_usd": 3591.58, "cost_per_trial_usd": 10.883575757575757, "seconds": 2351.5, "uncertainty": 3.09, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-xai-grok-4-6-none-grok-build.json", "cohort": "published-suite", "file": "2026-08-26-xai-grok-4-6-none-grok-build.json"}, {"id": "terminal-4:463aacbe832339a2", "model": "Gemini 3.8 Flash", "agent": "mini-SWE-agent", "agent_version": "2.4.6", "setting": "high", "score": 19.09, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-02", "cost_usd": 0.0, "cost_per_trial_usd": 0.0, "seconds": 1995.1, "uncertainty": 3.36, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent.json", "cohort": "published-suite", "file": "2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent.json"}, {"id": "terminal-4:c72ce12418d3714b", "model": "GPT-5.6 Luna", "agent": "Codex", "agent_version": "0.149.1", "setting": "max", "score": 17.27, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-06-26", "cost_usd": 346.67, "cost_per_trial_usd": 1.0505151515151516, "seconds": 4088.3, "uncertainty": 2.85, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-openai-gpt-5-6-luna-max-codex.json", "cohort": "published-suite", "file": "2026-08-26-openai-gpt-5-6-luna-max-codex.json"}, {"id": "terminal-4:7577d5c37f03b6ff", "model": "GPT-6 Luna", "agent": "Codex", "agent_version": "0.159.0", "setting": "max", "score": 16.36, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-22", "cost_usd": 72.14, "cost_per_trial_usd": 0.2186060606060606, "seconds": 2284.1, "uncertainty": 2.72, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-30-openai-gpt-6-luna-max-codex.json", "cohort": "published-suite", "file": "2026-09-30-openai-gpt-6-luna-max-codex.json"}, {"id": "terminal-4:5eeb3816cf02e6c0", "model": "Muse Spark 1.3", "agent": "Muse Code", "agent_version": "1.4.1", "setting": "xhigh", "score": 14.55, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-09-02", "cost_usd": 0.0, "cost_per_trial_usd": 0.0, "seconds": 4341.7, "uncertainty": 2.94, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-30-meta-muse-spark-1-3-xhigh-muse-code.json", "cohort": "published-suite", "file": "2026-09-30-meta-muse-spark-1-3-xhigh-muse-code.json"}, {"id": "terminal-4:fd637b8a757c4504", "model": "Grok 4.5", "agent": "Grok Build", "agent_version": "1.0.5", "setting": "none", "score": 12.42, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-07-16", "cost_usd": 2094.11, "cost_per_trial_usd": 6.345787878787879, "seconds": 3162.1, "uncertainty": 2.62, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-xai-grok-4-5-none-grok-build.json", "cohort": "published-suite", "file": "2026-08-26-xai-grok-4-5-none-grok-build.json"}, {"id": "terminal-4:c9fc6b719b237b8e", "model": "Sonnet 5", "agent": "Claude Code", "agent_version": "2.1.231", "setting": "max", "score": 12.42, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-06-30", "cost_usd": 9603.86, "cost_per_trial_usd": 29.102606060606064, "seconds": 6510.3, "uncertainty": 3.06, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-08-26-anthropic-claude-sonnet-5-max-claude-code.json", "cohort": "published-suite", "file": "2026-08-26-anthropic-claude-sonnet-5-max-claude-code.json"}, {"id": "terminal-4:11b804caa04ef17e", "model": "Gemini 3.7 Flash", "agent": "mini-SWE-agent", "agent_version": "2.4.6", "setting": "high", "score": 11.21, "first_attempt": null, "tasks": null, "trials": 330, "date": "2026-08-13", "cost_usd": 1261.87, "cost_per_trial_usd": 3.8238484848484844, "seconds": 1688.9, "uncertainty": 2.45, "uncertainty_kind": "95% CI half-width", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench/blob/bf4c1255fe70237aecd03a14b0fab9f01afca6b4/leaderboard/submissions/2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent.json", "cohort": "published-suite", "file": "2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent.json"}], "source": "https://github.com/harbor-framework/terminal-bench", "revision": "bf4c1255fe70237aecd03a14b0fab9f01afca6b4", "checksum": "8b0d48911038e8027496058c92fbb88b39862c2c381d50d4b1dfc402e637fe1b", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", "checked_at": "2026-10-08T08:26:25Z", "updated_at": "2026-10-08T08:26:25Z", "latest_result": "2026-09-29", "status": "ok", "status_message": "Source checked successfully", "cohorts": ["published-suite"]}, {"id": "terminal-21", "name": "Terminal-Bench 2.1", "metric": "Task success", "description": "A separate terminal task suite. Published accuracy; uncertainty is standard error, not a 95% confidence interval.", "historical": false, "rows": [{"id": "terminal-21:bd029cf248ced596", "model": "Fable 5", "agent": "Claude Code", "agent_version": "2.1.167", "setting": "xhigh", "score": 83.82, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-06-07", "cost_usd": 552.67, "cost_per_trial_usd": 1.2419550561797752, "seconds": 579.7, "uncertainty": 1.16, "uncertainty_kind": "Standard error", "disqualified_trials": 1, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-06-07-anthropic-claude-fable-5-xhigh-claude-code.json", "cohort": "published-suite", "file": "2026-06-07-anthropic-claude-fable-5-xhigh-claude-code.json"}, {"id": "terminal-21:33721cc09c0885de", "model": "GPT-5.5", "agent": "Codex", "agent_version": "0.125.0", "setting": "xhigh", "score": 83.15, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-01", "cost_usd": 2059.19, "cost_per_trial_usd": 4.627393258426967, "seconds": 482.6, "uncertainty": 1.13, "uncertainty_kind": "Standard error", "disqualified_trials": 1, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-openai-gpt-5-5-xhigh-codex.json", "cohort": "published-suite", "file": "2026-05-01-openai-gpt-5-5-xhigh-codex.json"}, {"id": "terminal-21:8a486d7d135b01f0", "model": "Fable 5", "agent": "Terminus 2", "agent_version": "2.0.0", "setting": "high", "score": 80.45, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-06-05", "cost_usd": 438.64, "cost_per_trial_usd": 0.9857078651685393, "seconds": 697.1, "uncertainty": 1.16, "uncertainty_kind": "Standard error", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-06-05-anthropic-claude-fable-5-high-terminus-2.json", "cohort": "published-suite", "file": "2026-06-05-anthropic-claude-fable-5-high-terminus-2.json"}, {"id": "terminal-21:29d21301547948aa", "model": "Grok 4.5", "agent": "Cursor CLI", "agent_version": "2026.07.08-0c04a8a", "setting": "high", "score": 79.33, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-09", "cost_usd": 134.09, "cost_per_trial_usd": 0.3013258426966292, "seconds": 443.0, "uncertainty": 1.46, "uncertainty_kind": "Standard error", "disqualified_trials": 40, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-09-cursor-grok-4-5-none-cursor-cli.json", "cohort": "published-suite", "file": "2026-07-09-cursor-grok-4-5-none-cursor-cli.json"}, {"id": "terminal-21:aac686e3e4cfef0e", "model": "Opus 4.8", "agent": "Claude Code", "agent_version": "2.1.205", "setting": "high", "score": 78.88, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-09", "cost_usd": 286.94, "cost_per_trial_usd": 0.6448089887640449, "seconds": 551.4, "uncertainty": 1.31, "uncertainty_kind": "Standard error", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-09-anthropic-claude-opus-4-8-high-claude-code.json", "cohort": "published-suite", "file": "2026-07-09-anthropic-claude-opus-4-8-high-claude-code.json"}, {"id": "terminal-21:db85396193da6a27", "model": "GPT-5.6 Terra", "agent": "Codex", "agent_version": "0.144.1", "setting": "max", "score": 78.43, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-11", "cost_usd": 421.15, "cost_per_trial_usd": 0.9464044943820225, "seconds": 432.8, "uncertainty": 1.25, "uncertainty_kind": "Standard error", "disqualified_trials": 1, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-11-openai-gpt-5-6-terra-max-codex.json", "cohort": "published-suite", "file": "2026-07-11-openai-gpt-5-6-terra-max-codex.json"}, {"id": "terminal-21:448f43c29e7711f3", "model": "GPT-5.5", "agent": "Terminus 2", "agent_version": "2.0.0", "setting": "xhigh", "score": 77.98, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-01", "cost_usd": 493.85, "cost_per_trial_usd": 1.1097752808988766, "seconds": 726.7, "uncertainty": 1.22, "uncertainty_kind": "Standard error", "disqualified_trials": 1, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-openai-gpt-5-5-xhigh-terminus-2.json", "cohort": "published-suite", "file": "2026-05-01-openai-gpt-5-5-xhigh-terminus-2.json"}, {"id": "terminal-21:e9c5ea9990dc3ec1", "model": "GPT-5.6 Sol", "agent": "Codex", "agent_version": "0.144.0", "setting": "max", "score": 76.18, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-10", "cost_usd": 574.68, "cost_per_trial_usd": 1.2914157303370786, "seconds": 431.0, "uncertainty": 1.28, "uncertainty_kind": "Standard error", "disqualified_trials": 32, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-10-gpt-5-6-sol-max-codex.json", "cohort": "published-suite", "file": "2026-07-10-gpt-5-6-sol-max-codex.json"}, {"id": "terminal-21:631d2af9085b2677", "model": "Muse Spark 1.1", "agent": "mini-SWE-agent", "agent_version": "2.4.5", "setting": "xhigh", "score": 76.18, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-09", "cost_usd": 198.05, "cost_per_trial_usd": 0.44505617977528095, "seconds": 687.0, "uncertainty": 1.23, "uncertainty_kind": "Standard error", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-09-openai-muse-spark-1-1-xhigh-mini-swe-agent.json", "cohort": "published-suite", "file": "2026-07-09-openai-muse-spark-1-1-xhigh-mini-swe-agent.json"}, {"id": "terminal-21:4dc80d6e749b52ca", "model": "GPT-5.6 Luna", "agent": "Codex", "agent_version": "0.144.1", "setting": "max", "score": 75.73, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-11", "cost_usd": 241.45, "cost_per_trial_usd": 0.5425842696629213, "seconds": 457.3, "uncertainty": 1.32, "uncertainty_kind": "Standard error", "disqualified_trials": 4, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-11-openai-gpt-5-6-luna-max-codex.json", "cohort": "published-suite", "file": "2026-07-11-openai-gpt-5-6-luna-max-codex.json"}, {"id": "terminal-21:76af9a227c52d704", "model": "Sonnet 5", "agent": "Claude Code", "agent_version": "2.1.205", "setting": "high", "score": 74.61, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-09", "cost_usd": 288.18, "cost_per_trial_usd": 0.6475955056179775, "seconds": 629.6, "uncertainty": 1.64, "uncertainty_kind": "Standard error", "disqualified_trials": 3, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-09-anthropic-claude-sonnet-5-high-claude-code.json", "cohort": "published-suite", "file": "2026-07-09-anthropic-claude-sonnet-5-high-claude-code.json"}, {"id": "terminal-21:650a542ab39ad99a", "model": "GPT-5.6 Terra", "agent": "Codex", "agent_version": "0.144.0", "setting": "max", "score": 74.38, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-10", "cost_usd": 401.03, "cost_per_trial_usd": 0.901191011235955, "seconds": 421.6, "uncertainty": 1.54, "uncertainty_kind": "Standard error", "disqualified_trials": 32, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-10-gpt-5-6-terra-max-codex.json", "cohort": "published-suite", "file": "2026-07-10-gpt-5-6-terra-max-codex.json"}, {"id": "terminal-21:f2f3a434633e0542", "model": "Gemini 3 Pro", "agent": "Terminus 2", "agent_version": "2.0.0", "setting": "high", "score": 73.93, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-01", "cost_usd": 224.44, "cost_per_trial_usd": 0.5043595505617977, "seconds": 547.8, "uncertainty": 1.29, "uncertainty_kind": "Standard error", "disqualified_trials": 2, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-gemini-gemini-3-pro-preview-high-terminus-2.json", "cohort": "published-suite", "file": "2026-05-01-gemini-gemini-3-pro-preview-high-terminus-2.json"}, {"id": "terminal-21:fda4b8d8cdaa5092", "model": "GPT-5.6 Luna", "agent": "Codex", "agent_version": "0.144.0", "setting": "max", "score": 71.24, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-07-10", "cost_usd": 218.28, "cost_per_trial_usd": 0.49051685393258426, "seconds": 422.0, "uncertainty": 1.39, "uncertainty_kind": "Standard error", "disqualified_trials": 33, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-07-10-gpt-5-6-luna-max-codex.json", "cohort": "published-suite", "file": "2026-07-10-gpt-5-6-luna-max-codex.json"}, {"id": "terminal-21:da0de26607a00ae8", "model": "Opus 4.7", "agent": "Claude Code", "agent_version": "2.1.123", "setting": "max", "score": 68.9, "first_attempt": null, "tasks": null, "trials": 447, "date": "2026-05-01", "cost_usd": 599.52, "cost_per_trial_usd": 1.3412080536912752, "seconds": 799.6, "uncertainty": 1.41, "uncertainty_kind": "Standard error", "disqualified_trials": 2, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-anthropic-claude-opus-4-7-max-claude-code.json", "cohort": "published-suite", "file": "2026-05-01-anthropic-claude-opus-4-7-max-claude-code.json"}, {"id": "terminal-21:febfe9182db689b8", "model": "Opus 4.7", "agent": "Terminus 2", "agent_version": "2.0.0", "setting": "max", "score": 66.07, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-01", "cost_usd": 582.26, "cost_per_trial_usd": 1.3084494382022471, "seconds": 1042.7, "uncertainty": 1.37, "uncertainty_kind": "Standard error", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-anthropic-claude-opus-4-7-max-terminus-2.json", "cohort": "published-suite", "file": "2026-05-01-anthropic-claude-opus-4-7-max-terminus-2.json"}, {"id": "terminal-21:686712792a878c08", "model": "Gemini 3 Pro", "agent": "Gemini CLI", "agent_version": "0.40.0", "setting": "high", "score": 65.84, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-01", "cost_usd": 247.76, "cost_per_trial_usd": 0.5567640449438203, "seconds": 436.6, "uncertainty": 1.38, "uncertainty_kind": "Standard error", "disqualified_trials": 2, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-gemini-gemini-3-pro-preview-high-gemini-cli.json", "cohort": "published-suite", "file": "2026-05-01-gemini-gemini-3-pro-preview-high-gemini-cli.json"}, {"id": "terminal-21:066839c09a9beced", "model": "Gemini 3.1 Pro", "agent": "Gemini CLI", "agent_version": "0.40.0", "setting": "high", "score": 65.84, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-05", "cost_usd": 236.49, "cost_per_trial_usd": 0.5314382022471911, "seconds": 652.9, "uncertainty": 1.67, "uncertainty_kind": "Standard error", "disqualified_trials": 1, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-05-gemini-gemini-3-1-pro-preview-high-gemini-cli.json", "cohort": "published-suite", "file": "2026-05-05-gemini-gemini-3-1-pro-preview-high-gemini-cli.json"}, {"id": "terminal-21:ee4ef6abf5c4f208", "model": "Gemini 3.1 Pro", "agent": "Terminus 2", "agent_version": "2.0.0", "setting": "high", "score": 65.62, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-05", "cost_usd": 229.99, "cost_per_trial_usd": 0.5168314606741573, "seconds": 672.8, "uncertainty": 1.65, "uncertainty_kind": "Standard error", "disqualified_trials": 2, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-05-gemini-gemini-3-1-pro-preview-high-terminus-2.json", "cohort": "published-suite", "file": "2026-05-05-gemini-gemini-3-1-pro-preview-high-terminus-2.json"}, {"id": "terminal-21:b7be84733d082beb", "model": "GLM-5.1", "agent": "Claude Code", "agent_version": "2.1.123", "setting": "max", "score": 58.65, "first_attempt": null, "tasks": null, "trials": 445, "date": "2026-05-01", "cost_usd": 277.14, "cost_per_trial_usd": 0.6227865168539326, "seconds": 944.6, "uncertainty": 1.24, "uncertainty_kind": "Standard error", "disqualified_trials": 0, "source": "https://github.com/harbor-framework/terminal-bench-2-1/blob/7131e4375048a0e408a8fb404b5f499d726b695b/leaderboard/submissions/2026-05-01-glm-5-1-max-claude-code.json", "cohort": "published-suite", "file": "2026-05-01-glm-5-1-max-claude-code.json"}], "source": "https://github.com/harbor-framework/terminal-bench-2-1", "revision": "7131e4375048a0e408a8fb404b5f499d726b695b", "checksum": "64603d9931c61f92b77bb33caa82ae45463354c3b3228a8ea3b701e91f2cd6c2", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", "checked_at": "2026-10-08T08:27:43Z", "updated_at": "2026-10-08T08:27:43Z", "latest_result": "2026-07-11", "status": "ok", "status_message": "Source checked successfully", "cohorts": ["published-suite"]}, {"id": "aider-polyglot", "name": "Aider Polyglot", "metric": "Pass rate after 2 attempts", "description": "Code editing across multiple programming languages using Aider. Historical runs; filter by task count before comparing.", "historical": true, "rows": [{"id": "aider-polyglot:c552cfecc2591f3c", "model": "gpt-5 (high)", "agent": "Aider", "agent_version": "0.86.2.dev", "setting": "diff", "score": 88.0, "first_attempt": 52.0, "tasks": 225, "trials": null, "date": "2025-08-23", "cost_usd": 29.0829, "cost_per_trial_usd": null, "seconds": 194.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-08-23-15-47-21--gpt-5-high"}, {"id": "aider-polyglot:4b114066b7dde21f", "model": "gpt-5 (medium)", "agent": "Aider", "agent_version": "0.86.2.dev", "setting": "diff", "score": 86.7, "first_attempt": 49.8, "tasks": 225, "trials": null, "date": "2025-08-25", "cost_usd": 17.693, "cost_per_trial_usd": null, "seconds": 118.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-08-25-13-23-27--gpt-5-medium"}, {"id": "aider-polyglot:4ef83cbfed7c85d6", "model": "o3-pro (high)", "agent": "Aider", "agent_version": "0.85.1.dev", "setting": "diff", "score": 84.9, "first_attempt": 43.6, "tasks": 225, "trials": null, "date": "2025-06-28", "cost_usd": 146.3249, "cost_per_trial_usd": null, "seconds": 449.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-28-00-38-18--o3-pro-high"}, {"id": "aider-polyglot:006937df70992ab0", "model": "gemini-2.5-pro-preview-06-05 (32k think)", "agent": "Aider", "agent_version": "0.84.1.dev", "setting": "diff-fenced", "score": 83.1, "first_attempt": 46.2, "tasks": 225, "trials": null, "date": "2025-06-06", "cost_usd": 49.8822, "cost_per_trial_usd": null, "seconds": 200.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced"}, {"id": "aider-polyglot:102fb983907532c4", "model": "gpt-5 (low)", "agent": "Aider", "agent_version": "0.86.2.dev", "setting": "diff", "score": 81.3, "first_attempt": 43.1, "tasks": 225, "trials": null, "date": "2025-08-25", "cost_usd": 10.3713, "cost_per_trial_usd": null, "seconds": 62.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-08-25-14-16-37--gpt-5-low"}, {"id": "aider-polyglot:6ab3d1d55a32f513", "model": "o3 (high)", "agent": "Aider", "agent_version": "0.84.1.dev", "setting": "diff", "score": 81.3, "first_attempt": 40.0, "tasks": 225, "trials": null, "date": "2025-06-25", "cost_usd": 21.2259, "cost_per_trial_usd": null, "seconds": 197.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-25-21-04-24--o3-price-reduction-high"}, {"id": "aider-polyglot:345dd1ba0c1dc6dd", "model": "grok-4 (high)", "agent": "Aider", "agent_version": "0.85.2.dev", "setting": "diff", "score": 79.6, "first_attempt": 40.9, "tasks": 225, "trials": null, "date": "2025-07-11", "cost_usd": 59.6182, "cost_per_trial_usd": null, "seconds": 403.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-07-11-19-37-40--xai-or-grok4-high"}, {"id": "aider-polyglot:408979decde7db9e", "model": "gemini-2.5-pro-preview-06-05 (default think)", "agent": "Aider", "agent_version": "0.84.1.dev", "setting": "diff-fenced", "score": 79.1, "first_attempt": 44.9, "tasks": 225, "trials": null, "date": "2025-06-06", "cost_usd": 45.5961, "cost_per_trial_usd": null, "seconds": 175.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-06-18-38-56--gemini0605-diff-fenced"}, {"id": "aider-polyglot:bce4034a5f0788c4", "model": "o3 (high) + gpt-4.1", "agent": "Aider", "agent_version": "0.85.1.dev", "setting": "architect", "score": 78.2, "first_attempt": 34.8, "tasks": 224, "trials": null, "date": "2025-06-27", "cost_usd": 17.5518, "cost_per_trial_usd": null, "seconds": 121.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "224", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-27-23-53-57--o3-mini-high-diff-arch"}, {"id": "aider-polyglot:4f670a1faa0d032d", "model": "Gemini 2.5 Pro Preview 05-06", "agent": "Aider", "agent_version": "0.82.4.dev", "setting": "diff-fenced", "score": 76.9, "first_attempt": 36.4, "tasks": 225, "trials": null, "date": "2025-05-07", "cost_usd": 37.4104, "cost_per_trial_usd": null, "seconds": 165.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost"}, {"id": "aider-polyglot:b4c12b58bb7fc911", "model": "o3", "agent": "Aider", "agent_version": "0.84.1.dev", "setting": "diff", "score": 76.9, "first_attempt": 40.9, "tasks": 225, "trials": null, "date": "2025-06-25", "cost_usd": 13.7517, "cost_per_trial_usd": null, "seconds": 101.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-25-20-30-16--o3-price-reduction"}, {"id": "aider-polyglot:9d623b6f1cd749c8", "model": "DeepSeek-V3.2-Exp (Reasoner)", "agent": "Aider", "agent_version": "0.86.2.dev", "setting": "diff", "score": 74.2, "first_attempt": 39.6, "tasks": 225, "trials": null, "date": "2025-10-03", "cost_usd": 1.3045, "cost_per_trial_usd": null, "seconds": 291.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-10-03-09-45-34--deepseek-v3.2-reasoner"}, {"id": "aider-polyglot:1e877385d232ffac", "model": "Gemini 2.5 Pro Preview 03-25", "agent": "Aider", "agent_version": "0.81.3.dev", "setting": "diff-fenced", "score": 72.9, "first_attempt": 40.9, "tasks": 225, "trials": null, "date": "2025-04-12", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 45.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-12-04-55-50--gemini-25-pro-diff-fenced"}, {"id": "aider-polyglot:912aaf737dc5c1df", "model": "claude-opus-4-20250514 (32k thinking)", "agent": "Aider", "agent_version": "0.83.3.dev", "setting": "diff", "score": 72.0, "first_attempt": 37.3, "tasks": 225, "trials": null, "date": "2025-05-25", "cost_usd": 65.7484, "cost_per_trial_usd": null, "seconds": 44.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-25-20-40-51--opus4-diff-exuser"}, {"id": "aider-polyglot:77ad69a4e98121e1", "model": "o4-mini (high)", "agent": "Aider", "agent_version": "0.82.1.dev", "setting": "diff", "score": 72.0, "first_attempt": 19.6, "tasks": 225, "trials": null, "date": "2025-04-16", "cost_usd": 19.6399, "cost_per_trial_usd": null, "seconds": 176.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-16-22-01-58--o4-mini-high-diff-exsys"}, {"id": "aider-polyglot:8816accab82d6804", "model": "DeepSeek R1 (0528)", "agent": "Aider", "agent_version": "0.84.1.dev", "setting": "diff", "score": 71.4, "first_attempt": 34.4, "tasks": 224, "trials": null, "date": "2025-06-06", "cost_usd": 4.8016, "cost_per_trial_usd": null, "seconds": 716.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "224", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-06-06-16-47-07--r1-diff"}, {"id": "aider-polyglot:65b8a066ba06551a", "model": "claude-opus-4-20250514 (no think)", "agent": "Aider", "agent_version": "0.83.3.dev", "setting": "diff", "score": 70.7, "first_attempt": 32.9, "tasks": 225, "trials": null, "date": "2025-05-25", "cost_usd": 68.6253, "cost_per_trial_usd": null, "seconds": 42.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-25-19-57-20--opus4-diff-exuser"}, {"id": "aider-polyglot:b944ea7eee092251", "model": "DeepSeek-V3.2-Exp (Chat)", "agent": "Aider", "agent_version": "0.86.2.dev", "setting": "diff", "score": 70.2, "first_attempt": 38.7, "tasks": 225, "trials": null, "date": "2025-10-03", "cost_usd": 0.8756, "cost_per_trial_usd": null, "seconds": 104.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-10-03-09-21-36--deepseek-v3.2-chat"}, {"id": "aider-polyglot:ef7e5c1fa9722f1f", "model": "claude-3-7-sonnet-20250219 (32k thinking tokens)", "agent": "Aider", "agent_version": "0.75.1.dev", "setting": "diff", "score": 64.9, "first_attempt": 29.3, "tasks": 225, "trials": null, "date": "2025-02-24", "cost_usd": 36.8343, "cost_per_trial_usd": null, "seconds": 105.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-02-24-21-47-23--sonnet37-diff-think-32k-64k"}, {"id": "aider-polyglot:b1e891592f06acea", "model": "DeepSeek R1 + claude-3-5-sonnet-20241022", "agent": "Aider", "agent_version": "0.72.3.dev", "setting": "architect", "score": 64.0, "first_attempt": 27.1, "tasks": 225, "trials": null, "date": "2025-01-23", "cost_usd": 13.2933, "cost_per_trial_usd": null, "seconds": 251.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-23-19-14-48--r1-architect-sonnet"}, {"id": "aider-polyglot:c7a1ad85f2d6c5f2", "model": "o1-2024-12-17 (high)", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "diff", "score": 61.7, "first_attempt": 23.7, "tasks": 224, "trials": null, "date": "2024-12-21", "cost_usd": 186.4958, "cost_per_trial_usd": null, "seconds": 133.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "224", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-21-19-23-03--polyglot-o1-hard-diff"}, {"id": "aider-polyglot:bfb973228267eed4", "model": "claude-sonnet-4-20250514 (32k thinking)", "agent": "Aider", "agent_version": "0.83.3.dev", "setting": "diff", "score": 61.3, "first_attempt": 25.8, "tasks": 225, "trials": null, "date": "2025-05-24", "cost_usd": 26.5755, "cost_per_trial_usd": null, "seconds": 79.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-24-22-10-36--sonnet4-diff-exuser-think32k"}, {"id": "aider-polyglot:230875c1b7e2d69a", "model": "claude-3-7-sonnet-20250219 (no thinking)", "agent": "Aider", "agent_version": "0.74.4.dev", "setting": "diff", "score": 60.4, "first_attempt": 24.4, "tasks": 225, "trials": null, "date": "2025-02-24", "cost_usd": 17.7191, "cost_per_trial_usd": null, "seconds": 28.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-02-24-19-54-07--sonnet37-diff"}, {"id": "aider-polyglot:14fd413937854412", "model": "o3-mini (high)", "agent": "Aider", "agent_version": "0.72.4.dev", "setting": "diff", "score": 60.4, "first_attempt": 21.0, "tasks": 224, "trials": null, "date": "2025-01-31", "cost_usd": 18.1584, "cost_per_trial_usd": null, "seconds": 124.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "224", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-31-20-42-47--o3-mini-diff-high"}, {"id": "aider-polyglot:1806e05880346cfd", "model": "Qwen3 235B A22B diff, no think, Alibaba API", "agent": "Aider", "agent_version": "0.82.4.dev", "setting": "diff", "score": 59.6, "first_attempt": 28.9, "tasks": 225, "trials": null, "date": "2025-05-09", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 45.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff"}, {"id": "aider-polyglot:d641506d457c2b10", "model": "Kimi K2", "agent": "Aider", "agent_version": "0.85.3.dev", "setting": "diff", "score": 59.1, "first_attempt": 20.4, "tasks": 225, "trials": null, "date": "2025-07-17", "cost_usd": 1.2357, "cost_per_trial_usd": null, "seconds": 67.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-07-17-17-41-54--kimi-k2-diff-or-pricing"}, {"id": "aider-polyglot:62c360ef0886d44e", "model": "DeepSeek R1", "agent": "Aider", "agent_version": "0.71.2.dev", "setting": "diff", "score": 56.9, "first_attempt": 26.7, "tasks": 225, "trials": null, "date": "2025-01-20", "cost_usd": 5.4193, "cost_per_trial_usd": null, "seconds": 113.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer"}, {"id": "aider-polyglot:4bcbd0ff1d35f6d7", "model": "claude-sonnet-4-20250514 (no thinking)", "agent": "Aider", "agent_version": "0.83.3.dev", "setting": "diff", "score": 56.4, "first_attempt": 20.4, "tasks": 225, "trials": null, "date": "2025-05-24", "cost_usd": 15.8155, "cost_per_trial_usd": null, "seconds": 29.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-24-21-17-54--sonnet4-diff-exuser"}, {"id": "aider-polyglot:ee76acaa7cf1c23c", "model": "DeepSeek V3 (0324)", "agent": "Aider", "agent_version": "0.78.1.dev", "setting": "diff", "score": 55.1, "first_attempt": 28.0, "tasks": 225, "trials": null, "date": "2025-03-24", "cost_usd": 1.1164, "cost_per_trial_usd": null, "seconds": 290.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff"}, {"id": "aider-polyglot:04fc642c0eccb74d", "model": "gemini-2.5-flash-preview-05-20 (24k think)", "agent": "Aider", "agent_version": "0.83.3.dev", "setting": "diff", "score": 55.1, "first_attempt": 26.2, "tasks": 225, "trials": null, "date": "2025-05-25", "cost_usd": 8.5625, "cost_per_trial_usd": null, "seconds": 53.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-25-22-58-44--flash25-05-20-24k-think"}, {"id": "aider-polyglot:5336aa72794204b2", "model": "Quasar Alpha", "agent": "Aider", "agent_version": "0.80.5.dev", "setting": "diff", "score": 54.7, "first_attempt": 21.8, "tasks": 225, "trials": null, "date": "2025-04-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 14.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-04-02-57-25--qalpha-diff-exsys"}, {"id": "aider-polyglot:c8f7925f2707c689", "model": "o3-mini (medium)", "agent": "Aider", "agent_version": "0.72.4.dev", "setting": "diff", "score": 53.8, "first_attempt": 19.1, "tasks": 225, "trials": null, "date": "2025-01-31", "cost_usd": 8.8599, "cost_per_trial_usd": null, "seconds": 47.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-31-20-27-46--o3-mini-diff2"}, {"id": "aider-polyglot:4d22ee37aa8df07b", "model": "Grok 3 Beta", "agent": "Aider", "agent_version": "0.81.2.dev", "setting": "diff", "score": 53.3, "first_attempt": 22.2, "tasks": 225, "trials": null, "date": "2025-04-10", "cost_usd": 11.0338, "cost_per_trial_usd": null, "seconds": 15.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-10-04-21-31--grok3-diff-exuser"}, {"id": "aider-polyglot:0b467e032225b103", "model": "Optimus Alpha", "agent": "Aider", "agent_version": "0.81.2.dev", "setting": "diff", "score": 52.9, "first_attempt": 21.3, "tasks": 225, "trials": null, "date": "2025-04-10", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-10-19-02-44--oalpha-diff-exsys"}, {"id": "aider-polyglot:4a52e7d43a84b596", "model": "gpt-4.1", "agent": "Aider", "agent_version": "0.81.4.dev", "setting": "diff", "score": 52.4, "first_attempt": 20.0, "tasks": 225, "trials": null, "date": "2025-04-14", "cost_usd": 9.8556, "cost_per_trial_usd": null, "seconds": 20.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-14-21-05-54--gpt41-diff-exuser"}, {"id": "aider-polyglot:2fa8416b721ccf11", "model": "claude-3-5-sonnet-20241022", "agent": "Aider", "agent_version": "0.71.2.dev", "setting": "diff", "score": 51.6, "first_attempt": 22.2, "tasks": 225, "trials": null, "date": "2025-01-17", "cost_usd": 14.4063, "cost_per_trial_usd": null, "seconds": 21.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-17-19-44-33--sonnet-baseline-jan-17"}, {"id": "aider-polyglot:429774706591af14", "model": "Grok 3 Mini Beta (high)", "agent": "Aider", "agent_version": "0.81.3.dev", "setting": "whole", "score": 49.3, "first_attempt": 17.3, "tasks": 225, "trials": null, "date": "2025-04-10", "cost_usd": 0.7346, "cost_per_trial_usd": null, "seconds": 79.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-10-23-59-02--xai-grok3-mini-whole-high"}, {"id": "aider-polyglot:5adfbc9ccde37a3e", "model": "DeepSeek Chat V3 (prev)", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "diff", "score": 48.4, "first_attempt": 22.7, "tasks": 225, "trials": null, "date": "2024-12-25", "cost_usd": 0.3369, "cost_per_trial_usd": null, "seconds": 34.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-25-13-31-51--deepseekv3preview-diff2"}, {"id": "aider-polyglot:25242445fcfa3283", "model": "gemini-2.5-flash-preview-04-17 (default)", "agent": "Aider", "agent_version": "0.82.3.dev", "setting": "diff", "score": 47.1, "first_attempt": 21.8, "tasks": 225, "trials": null, "date": "2025-04-20", "cost_usd": 1.8451, "cost_per_trial_usd": null, "seconds": 50.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-20-19-54-31--flash25-diff-no-think"}, {"id": "aider-polyglot:a41eb9bbd37a7dd3", "model": "chatgpt-4o-latest (2025-03-29)", "agent": "Aider", "agent_version": "0.79.3.dev", "setting": "diff", "score": 45.3, "first_attempt": 16.4, "tasks": 225, "trials": null, "date": "2025-03-29", "cost_usd": 19.7416, "cost_per_trial_usd": null, "seconds": 10.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-03-29-05-24-55--chatgpt4o-mar28-diff"}, {"id": "aider-polyglot:4fcc5b5f013e007f", "model": "gpt-4.5-preview", "agent": "Aider", "agent_version": "0.75.2.dev", "setting": "diff", "score": 44.9, "first_attempt": 22.3, "tasks": 224, "trials": null, "date": "2025-02-27", "cost_usd": 183.1802, "cost_per_trial_usd": null, "seconds": 113.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "224", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-02-27-20-26-15--gpt45-diff3"}, {"id": "aider-polyglot:e106eb45b8ed9156", "model": "gemini-2.5-flash-preview-05-20 (no think)", "agent": "Aider", "agent_version": "0.83.3.dev", "setting": "diff", "score": 44.0, "first_attempt": 20.9, "tasks": 225, "trials": null, "date": "2025-05-26", "cost_usd": 1.1354, "cost_per_trial_usd": null, "seconds": 12.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-26-15-56-31--flash25-05-20-24k-think"}, {"id": "aider-polyglot:7e9bac23a4a675d3", "model": "gpt-oss-120b (high)", "agent": "Aider", "agent_version": "0.85.3.dev", "setting": "diff", "score": 41.8, "first_attempt": 13.8, "tasks": 225, "trials": null, "date": "2025-08-06", "cost_usd": 0.7406, "cost_per_trial_usd": null, "seconds": 35.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-08-06-04-54-48--gpt-oss-120b-high-polyglot"}, {"id": "aider-polyglot:95fc7325f57850a8", "model": "Qwen3 32B", "agent": "Aider", "agent_version": "0.82.4.dev", "setting": "diff", "score": 40.0, "first_attempt": 14.2, "tasks": 225, "trials": null, "date": "2025-05-08", "cost_usd": 0.7603, "cost_per_trial_usd": null, "seconds": 372.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-05-08-03-20-24--qwen3-32b-default"}, {"id": "aider-polyglot:7fc8aa55680413e8", "model": "gemini-exp-1206", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 38.2, "first_attempt": 19.6, "tasks": 225, "trials": null, "date": "2024-12-22", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 45.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2"}, {"id": "aider-polyglot:b11f9fa01b223c6a", "model": "Gemini 2.0 Pro exp-02-05", "agent": "Aider", "agent_version": "0.75.2.dev", "setting": "whole", "score": 35.6, "first_attempt": 20.4, "tasks": 225, "trials": null, "date": "2025-02-25", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 34.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-02-25-20-23-07--gemini-pro"}, {"id": "aider-polyglot:c253f2d627cb8f5f", "model": "Grok 3 Mini Beta (low)", "agent": "Aider", "agent_version": "0.81.2.dev", "setting": "whole", "score": 34.7, "first_attempt": 11.1, "tasks": 225, "trials": null, "date": "2025-04-10", "cost_usd": 0.7856, "cost_per_trial_usd": null, "seconds": 35.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-10-18-47-24--grok3-mini-whole-exuser"}, {"id": "aider-polyglot:e47909a7d93514fa", "model": "o1-mini-2024-09-12", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 32.9, "first_attempt": 5.8, "tasks": 225, "trials": null, "date": "2024-12-22", "cost_usd": 18.577, "cost_per_trial_usd": null, "seconds": 34.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-22-21-26-35--polyglot-o1mini-whole"}, {"id": "aider-polyglot:f21d2bcd7efec439", "model": "gpt-4.1-mini", "agent": "Aider", "agent_version": "0.81.4.dev", "setting": "diff", "score": 32.4, "first_attempt": 11.1, "tasks": 225, "trials": null, "date": "2025-04-14", "cost_usd": 1.9918, "cost_per_trial_usd": null, "seconds": 19.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-14-21-27-53--gpt41mini-diff"}, {"id": "aider-polyglot:7cf35abbe3b439b0", "model": "claude-3-5-haiku-20241022", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "diff", "score": 28.0, "first_attempt": 7.1, "tasks": 225, "trials": null, "date": "2024-12-21", "cost_usd": 6.0583, "cost_per_trial_usd": null, "seconds": 31.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-21-21-46-27--polyglot-haiku-diff"}, {"id": "aider-polyglot:fea98cc773eb7e72", "model": "chatgpt-4o-latest (2025-02-15)", "agent": "Aider", "agent_version": "0.74.3.dev", "setting": "diff", "score": 27.1, "first_attempt": 9.0, "tasks": 223, "trials": null, "date": "2025-02-15", "cost_usd": 14.3703, "cost_per_trial_usd": null, "seconds": 12.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "223", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-02-15-19-51-22--chatgpt4o-feb15-diff"}, {"id": "aider-polyglot:83715cbcfbce0492", "model": "QwQ-32B + Qwen 2.5 Coder Instruct", "agent": "Aider", "agent_version": "0.75.3.dev", "setting": "architect", "score": 26.2, "first_attempt": 9.8, "tasks": 225, "trials": null, "date": "2025-03-07", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 137.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-03-07-15-11-27--qwq32b-arch-temp-topp-again"}, {"id": "aider-polyglot:5ce37490b8f38481", "model": "gpt-4o-2024-08-06", "agent": "Aider", "agent_version": "0.70.1.dev", "setting": "diff", "score": 23.1, "first_attempt": 4.9, "tasks": 225, "trials": null, "date": "2024-12-30", "cost_usd": 7.0286, "cost_per_trial_usd": null, "seconds": 16.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt"}, {"id": "aider-polyglot:91923b40924d9cab", "model": "gemini-2.0-flash-exp", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 22.2, "first_attempt": 11.6, "tasks": 225, "trials": null, "date": "2024-12-22", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 12.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole"}, {"id": "aider-polyglot:f343fc7eddebfbee", "model": "qwen-max-2025-01-25", "agent": "Aider", "agent_version": "0.72.4.dev", "setting": "diff", "score": 21.8, "first_attempt": 9.3, "tasks": 225, "trials": null, "date": "2025-01-28", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 39.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff"}, {"id": "aider-polyglot:44233b39b3a88767", "model": "QwQ-32B", "agent": "Aider", "agent_version": "0.75.3.dev", "setting": "diff", "score": 20.9, "first_attempt": 8.0, "tasks": 225, "trials": null, "date": "2025-03-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 228.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real"}, {"id": "aider-polyglot:93976aa93dc075cb", "model": "gemini-2.0-flash-thinking-exp-01-21", "agent": "Aider", "agent_version": "0.72.2.dev", "setting": "diff", "score": 18.2, "first_attempt": 5.8, "tasks": 225, "trials": null, "date": "2025-01-21", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 24.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff"}, {"id": "aider-polyglot:04f476044727e619", "model": "gpt-4o-2024-11-20", "agent": "Aider", "agent_version": "0.70.1.dev", "setting": "diff", "score": 18.2, "first_attempt": 4.9, "tasks": 225, "trials": null, "date": "2024-12-30", "cost_usd": 6.7351, "cost_per_trial_usd": null, "seconds": 12.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys"}, {"id": "aider-polyglot:a7b50670d8617762", "model": "DeepSeek Chat V2.5", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "diff", "score": 17.8, "first_attempt": 5.3, "tasks": 225, "trials": null, "date": "2024-12-21", "cost_usd": 0.5101, "cost_per_trial_usd": null, "seconds": 184.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-21-20-56-21--polyglot-deepseek-diff"}, {"id": "aider-polyglot:2f0d181140d94582", "model": "Qwen2.5-Coder-32B-Instruct", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 16.4, "first_attempt": 4.9, "tasks": 225, "trials": null, "date": "2024-12-26", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 42.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct"}, {"id": "aider-polyglot:f2155382d88c7a56", "model": "Llama 4 Maverick", "agent": "Aider", "agent_version": "0.81.2.dev", "setting": "whole", "score": 15.6, "first_attempt": 4.4, "tasks": 225, "trials": null, "date": "2025-04-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 20.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole"}, {"id": "aider-polyglot:25ef8416a086bdf9", "model": "yi-lightning", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 12.9, "first_attempt": 5.8, "tasks": 225, "trials": null, "date": "2024-12-23", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 146.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-23-01-11-56--yi-test"}, {"id": "aider-polyglot:d7b121afbaf2b789", "model": "command-a-03-2025-quality", "agent": "Aider", "agent_version": "0.77.1.dev", "setting": "whole", "score": 12.0, "first_attempt": 2.2, "tasks": 225, "trials": null, "date": "2025-03-14", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 85.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-03-14-23-40-00--cmda-quality-whole2"}, {"id": "aider-polyglot:81a21e8ed126c2a2", "model": "Codestral 25.01", "agent": "Aider", "agent_version": "0.71.2.dev", "setting": "whole", "score": 11.1, "first_attempt": 4.0, "tasks": 225, "trials": null, "date": "2025-01-13", "cost_usd": 1.9834, "cost_per_trial_usd": null, "seconds": 9.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-01-13-18-17-25--codestral-whole2"}, {"id": "aider-polyglot:10395a725f8cb9cb", "model": "openhands-lm-32b-v0.1", "agent": "Aider", "agent_version": "0.82.2.dev", "setting": "whole", "score": 10.2, "first_attempt": 4.0, "tasks": 225, "trials": null, "date": "2025-04-19", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 195.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-19-14-43-04--o4-mini-patch"}, {"id": "aider-polyglot:088a53596ce7918d", "model": "gpt-4.1-nano", "agent": "Aider", "agent_version": "0.81.4.dev", "setting": "whole", "score": 8.9, "first_attempt": 3.1, "tasks": 225, "trials": null, "date": "2025-04-14", "cost_usd": 0.4281, "cost_per_trial_usd": null, "seconds": 12.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-04-14-22-46-01--gpt41nano-diff"}, {"id": "aider-polyglot:965213a6d29ae14a", "model": "Qwen2.5-Coder-32B-Instruct", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "diff", "score": 8.0, "first_attempt": 4.4, "tasks": 225, "trials": null, "date": "2024-12-22", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 84.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-22-13-22-32--polyglot-qwen-diff"}, {"id": "aider-polyglot:4a0471aba501663f", "model": "gemma-3-27b-it", "agent": "Aider", "agent_version": "0.77.1.dev", "setting": "whole", "score": 4.9, "first_attempt": 1.8, "tasks": 225, "trials": null, "date": "2025-03-15", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 79.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2025-03-15-01-21-24--gemma3-27b-or"}, {"id": "aider-polyglot:7aca33f03b1afdf9", "model": "gpt-4o-mini-2024-07-18", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 3.6, "first_attempt": 0.9, "tasks": 225, "trials": null, "date": "2024-12-21", "cost_usd": 0.3236, "cost_per_trial_usd": null, "seconds": 17.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml", "file": "2024-12-21-18-41-18--polyglot-gpt-4o-mini"}], "source": "https://github.com/Aider-AI/aider", "revision": "5dc9490bb35f9729ef2c95d00a19ccd30c26339c", "checksum": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", "checked_at": "2026-10-08T08:27:44Z", "updated_at": "2026-10-08T08:27:44Z", "latest_result": "2025-10-03", "status": "ok", "status_message": "Source checked successfully", "cohorts": ["225", "224", "223"]}, {"id": "aider-edit", "name": "Aider Code Editing", "metric": "Pass rate after 2 attempts", "description": "Historical code editing runs using Aider. Task counts and editing formats vary between runs.", "historical": true, "rows": [{"id": "aider-edit:90f1b72b5105c641", "model": "claude-3-5-sonnet-20241022", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "diff", "score": 84.2, "first_attempt": 69.2, "tasks": 133, "trials": null, "date": "2024-10-22", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-22-17-45-28--sonnet-1022-diff-fixed-model-settings"}, {"id": "aider-edit:954f91a1e0ee77c5", "model": "o1", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "diff", "score": 84.2, "first_attempt": 65.4, "tasks": 133, "trials": null, "date": "2024-12-18", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 29.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-18-01-50-08--o1"}, {"id": "aider-edit:f496c6a6f3da069d", "model": "gemini-exp-1206 (whole)", "agent": "Aider", "agent_version": "0.67.1.dev", "setting": "whole", "score": 80.5, "first_attempt": 60.9, "tasks": 133, "trials": null, "date": "2024-12-08", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 64.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-08-21-39-06--gemini-exp-1206-whole"}, {"id": "aider-edit:f1bb76bb97833835", "model": "o1-preview", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "diff", "score": 79.7, "first_attempt": 57.9, "tasks": 133, "trials": null, "date": "2024-09-21", "cost_usd": 63.919, "cost_per_trial_usd": null, "seconds": 80.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-21-16-45-11--o1-preview-flex-sr-markers"}, {"id": "aider-edit:167a2303c3c44318", "model": "claude-3.5-sonnet-20240620", "agent": "Aider", "agent_version": "0.42.1-dev", "setting": "diff", "score": 77.4, "first_attempt": 57.1, "tasks": 133, "trials": null, "date": "2024-07-04", "cost_usd": 3.6346, "cost_per_trial_usd": null, "seconds": 17.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-04-14-32-08--claude-3.5-sonnet-diff-continue"}, {"id": "aider-edit:d30df6f01217cf7a", "model": "claude-3-5-haiku-20241022", "agent": "Aider", "agent_version": "0.61.1.dev", "setting": "diff", "score": 75.2, "first_attempt": 61.7, "tasks": 133, "trials": null, "date": "2024-11-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-04-19-19-32--haiku35-diff-ex-as-sys-false"}, {"id": "aider-edit:6dd82f894a2192f9", "model": "DeepSeek Coder V2 0724", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "diff", "score": 72.9, "first_attempt": 57.9, "tasks": 133, "trials": null, "date": "2024-07-24", "cost_usd": 0.0981, "cost_per_trial_usd": null, "seconds": 36.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-24-07-10-58--deepseek-coder2-0724-diff-direct"}, {"id": "aider-edit:dfcb62ae70acd10f", "model": "gpt-4o-2024-05-13", "agent": "Aider", "agent_version": "0.34.1-dev", "setting": "diff", "score": 72.9, "first_attempt": 60.2, "tasks": 133, "trials": null, "date": "2024-05-13", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 6.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-13-17-39-05--gpt-4o-diff"}, {"id": "aider-edit:4c0596838df3885a", "model": "ollama/qwen2.5-coder:32b", "agent": "Aider", "agent_version": "0.65.2.dev", "setting": "whole", "score": 72.9, "first_attempt": 58.6, "tasks": 133, "trials": null, "date": "2024-11-28", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 147.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-28-07-42-56--qwen2.5-coder-32b-whole-4"}, {"id": "aider-edit:5ecfcfa040b036f4", "model": "DeepSeek V2.5", "agent": "Aider", "agent_version": "0.55.1.dev", "setting": "diff", "score": 72.2, "first_attempt": 54.9, "tasks": 133, "trials": null, "date": "2024-09-05", "cost_usd": 0.0998, "cost_per_trial_usd": null, "seconds": 49.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-05-14-50-11--deepseek-sep5-no-shell"}, {"id": "aider-edit:0d4570c39eb63853", "model": "DeepSeek-V2.5-1210", "agent": "Aider", "agent_version": "0.67.1.dev", "setting": "diff", "score": 72.2, "first_attempt": 58.6, "tasks": 133, "trials": null, "date": "2024-12-10", "cost_usd": 0.1106, "cost_per_trial_usd": null, "seconds": 32.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-10-14-45-21--deepseek-1210-diff"}, {"id": "aider-edit:2be70132ee14fee8", "model": "openai/chatgpt-4o-latest", "agent": "Aider", "agent_version": "0.58.2.dev", "setting": "diff", "score": 72.2, "first_attempt": 56.4, "tasks": 133, "trials": null, "date": "2024-10-04", "cost_usd": 4.0641, "cost_per_trial_usd": null, "seconds": 23.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-04-16-30-08--chatgpt-4o-latest-diff-oct4"}, {"id": "aider-edit:aed0d3600fbf3afa", "model": "Qwen2.5-Coder-32B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "diff", "score": 71.4, "first_attempt": 59.4, "tasks": 133, "trials": null, "date": "2024-11-09", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 22.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-09-11-09-15--Qwen2.5-Coder-32B-Instruct"}, {"id": "aider-edit:09231ce17b469d66", "model": "gpt-4o-2024-08-06", "agent": "Aider", "agent_version": "0.48.1-dev", "setting": "diff", "score": 71.4, "first_attempt": 57.1, "tasks": 133, "trials": null, "date": "2024-08-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 6.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-08-06-18-28-39--gpt-4o-2024-08-06-diff-again"}, {"id": "aider-edit:f1267578aa17e2b3", "model": "gpt-4o-2024-11-20", "agent": "Aider", "agent_version": "0.63.3.dev", "setting": "diff", "score": 71.4, "first_attempt": 58.6, "tasks": 133, "trials": null, "date": "2024-11-20", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 6.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-20-19-28-30--gpt-4o-2024-11-20"}, {"id": "aider-edit:a5519de747f81b76", "model": "o1-mini (whole)", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "whole", "score": 70.7, "first_attempt": 49.6, "tasks": 133, "trials": null, "date": "2024-09-12", "cost_usd": 5.3725, "cost_per_trial_usd": null, "seconds": 103.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-12-19-57-35--o1-mini-whole"}, {"id": "aider-edit:76e3cc8e35185a7d", "model": "DeepSeek Chat V2 0628", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "diff", "score": 69.9, "first_attempt": 60.9, "tasks": 133, "trials": null, "date": "2024-07-19", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 37.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-19-08-57-13--openrouter-deepseek-chat-v2-0628"}, {"id": "aider-edit:df39c22eb133987b", "model": "gemini-2.0-flash-exp", "agent": "Aider", "agent_version": "0.68.1.dev", "setting": "diff", "score": 69.9, "first_attempt": 56.4, "tasks": 133, "trials": null, "date": "2024-12-11", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 7.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-11-21-07-00--gemini-2-flash-diff"}, {"id": "aider-edit:98e2fbe074af3daa", "model": "Qwen2.5-Coder-14B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 69.2, "first_attempt": 58.6, "tasks": 133, "trials": null, "date": "2024-10-29", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-29-11-53-39--Qwen2.5-Coder-14B-Instruct"}, {"id": "aider-edit:41b1bf3d1ceaa9bc", "model": "gemini-exp-1206 (diff)", "agent": "Aider", "agent_version": "0.67.1.dev", "setting": "diff", "score": 69.2, "first_attempt": 55.6, "tasks": 133, "trials": null, "date": "2024-12-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 32.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-06-21-35-50--gemini-exp-1206-diff"}, {"id": "aider-edit:7b1d4e1cb6da0e61", "model": "claude-3-opus-20240229", "agent": "Aider", "agent_version": "0.30.2-dev", "setting": "diff", "score": 68.4, "first_attempt": 53.4, "tasks": 133, "trials": null, "date": "2024-05-01", "cost_usd": 13.8395, "cost_per_trial_usd": null, "seconds": 32.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-01-20-05-59--direct-opus-filenames-outside-fence"}, {"id": "aider-edit:6b98e38db8f7a409", "model": "gpt-4-0613", "agent": "Aider", "agent_version": "0.18.2-dev", "setting": "diff", "score": 67.7, "first_attempt": 46.6, "tasks": 133, "trials": null, "date": "2023-12-16", "cost_usd": 17.4657, "cost_per_trial_usd": null, "seconds": 33.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2023-12-16-21-24-28--editblock-gpt-4-0613-actual-main"}, {"id": "aider-edit:e9256360ffaee634", "model": "Dracarys2-72B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 66.9, "first_attempt": 55.6, "tasks": 133, "trials": null, "date": "2024-10-05", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 46.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-05-20-03-10--dracarys-glhf-whole"}, {"id": "aider-edit:8d6a43458daf50bf", "model": "gemini-1.5-pro-exp-0827", "agent": "Aider", "agent_version": "0.53.1-dev", "setting": "diff-fenced", "score": 66.9, "first_attempt": 54.9, "tasks": 133, "trials": null, "date": "2024-08-28", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 14.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-08-28-07-10-50--gemini-1.5-pro-exp-0827-diff-fenced"}, {"id": "aider-edit:eaa764b344ace749", "model": "gpt-4-0125-preview", "agent": "Aider", "agent_version": "0.22.1-dev", "setting": "udiff", "score": 66.2, "first_attempt": 55.6, "tasks": 133, "trials": null, "date": "2024-01-25", "cost_usd": 14.6428, "cost_per_trial_usd": null, "seconds": 44.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-01-25-23-37-15--jan-exercism-gpt-4-0125-preview-udiff"}, {"id": "aider-edit:e4b6b60536420751", "model": "gpt-4-0314", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "diff", "score": 66.2, "first_attempt": 50.4, "tasks": 133, "trials": null, "date": "2024-05-04", "cost_usd": 16.2689, "cost_per_trial_usd": null, "seconds": 19.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-04-15-07-30--redo-gpt-4-0314-diff-reminder-rules"}, {"id": "aider-edit:d9b0c5fd0479494f", "model": "llama-3.1-405b-instruct (whole)", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "whole", "score": 66.2, "first_attempt": 48.9, "tasks": 133, "trials": null, "date": "2024-07-24", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-24-06-30-29--llama-405b-whole"}, {"id": "aider-edit:ef7196fec3bbda0b", "model": "Mistral Large (2411)", "agent": "Aider", "agent_version": "0.63.3.dev", "setting": "diff", "score": 65.4, "first_attempt": 46.6, "tasks": 133, "trials": null, "date": "2024-11-20", "cost_usd": 3.2334, "cost_per_trial_usd": null, "seconds": 24.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-20-14-57-11--mistral-2411-direct-diff"}, {"id": "aider-edit:ab75a5121dd2e25b", "model": "gemini-1.5-pro-002", "agent": "Aider", "agent_version": "0.57.2.dev", "setting": "diff-fenced", "score": 65.4, "first_attempt": 49.6, "tasks": 133, "trials": null, "date": "2024-09-24", "cost_usd": 2.8166, "cost_per_trial_usd": null, "seconds": 11.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-24-16-26-45--gemini-1.5-pro-002-diff-fenced"}, {"id": "aider-edit:4717b42a72d0f62a", "model": "gpt-4-1106-preview", "agent": "Aider", "agent_version": "0.33.1-dev", "setting": "udiff", "score": 65.4, "first_attempt": 51.9, "tasks": 133, "trials": null, "date": "2024-05-08", "cost_usd": 6.6061, "cost_per_trial_usd": null, "seconds": 20.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-08-21-16-03--may-gpt-4-1106-preview-udiff"}, {"id": "aider-edit:7c257bfd12db17d5", "model": "openrouter/qwen/qwen-2.5-coder-32b-instruct", "agent": "Aider", "agent_version": "0.63.3.dev", "setting": "diff", "score": 65.4, "first_attempt": 49.6, "tasks": 133, "trials": null, "date": "2024-11-20", "cost_usd": 0.1497, "cost_per_trial_usd": null, "seconds": 40.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-20-15-17-37--qwen25-32b-or-diff"}, {"id": "aider-edit:c77c931165beb347", "model": "qwen-2.5-72b-instruct (bf16)", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "diff", "score": 65.4, "first_attempt": 53.4, "tasks": 133, "trials": null, "date": "2024-09-20", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 39.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-20-20-20-19--qwen-2.5-72b-instruct-diff"}, {"id": "aider-edit:013f12953f98bb45", "model": "yi-lightning", "agent": "Aider", "agent_version": "0.67.1.dev", "setting": "whole", "score": 65.4, "first_attempt": 49.6, "tasks": 133, "trials": null, "date": "2024-12-11", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 57.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-11-00-37-08--yi-test"}, {"id": "aider-edit:c9a8c4956b3703c0", "model": "gpt-4-turbo-2024-04-09 (udiff)", "agent": "Aider", "agent_version": "0.30.2-dev", "setting": "udiff", "score": 63.9, "first_attempt": 48.1, "tasks": 133, "trials": null, "date": "2024-05-01", "cost_usd": 6.3337, "cost_per_trial_usd": null, "seconds": 22.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-01-02-09-20--gpt-4-turbo-examples"}, {"id": "aider-edit:7fad1f7fce86ecb5", "model": "llama-3.1-405b-instruct (diff)", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "diff", "score": 63.9, "first_attempt": 46.6, "tasks": 133, "trials": null, "date": "2024-07-23", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 56.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-23-22-07-08--llama-205b-diff"}, {"id": "aider-edit:f0217ad1a5d0731f", "model": "nousresearch/hermes-3-llama-3.1-405b", "agent": "Aider", "agent_version": "0.54.8-dev", "setting": "whole", "score": 63.9, "first_attempt": 51.1, "tasks": 133, "trials": null, "date": "2024-08-30", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 38.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-08-30-15-02-05--nous405b-whole"}, {"id": "aider-edit:6d472f7fa4ad5a92", "model": "ollama/Qwen2.5.1-Coder-7B-Instruct-GGUF:Q8_0-32k", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 63.9, "first_attempt": 52.6, "tasks": 133, "trials": null, "date": "2024-11-07", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-07-06-15-36--Qwen2.5.1-Coder-7B-Instruct-GGUF:Q8_0-32k-whole"}, {"id": "aider-edit:c6fc6d83de79287f", "model": "ollama/qwen2.5-coder:14b", "agent": "Aider", "agent_version": "0.65.2.dev", "setting": "whole", "score": 61.7, "first_attempt": 53.4, "tasks": 133, "trials": null, "date": "2024-11-27", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 58.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-27-07-41-51--qwen2.5-coder-14b-whole-1"}, {"id": "aider-edit:a6d4f754215831f9", "model": "o1-mini", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "diff", "score": 61.1, "first_attempt": 50.0, "tasks": 36, "trials": null, "date": "2024-09-21", "cost_usd": 2.4226, "cost_per_trial_usd": null, "seconds": 26.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "36", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-21-16-40-56--o1-mini-flex-sr-markers"}, {"id": "aider-edit:d7c52b94a37f8cc2", "model": "gemini-exp-1114", "agent": "Aider", "agent_version": "0.63.2.dev", "setting": "diff", "score": 60.9, "first_attempt": 50.4, "tasks": 133, "trials": null, "date": "2024-11-15", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 38.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-15-20-33-31--gemini-exp-1114-diff"}, {"id": "aider-edit:61247680ebf466fe", "model": "Mistral Large 2 (2407)", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "whole", "score": 60.2, "first_attempt": 39.8, "tasks": 133, "trials": null, "date": "2024-07-24", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 26.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-24-19-08-47--mistral-large-2407-whole"}, {"id": "aider-edit:3ca696eb59543453", "model": "llama-3.3-70b-instruct", "agent": "Aider", "agent_version": "0.67.1.dev", "setting": "diff", "score": 59.4, "first_attempt": 42.1, "tasks": 133, "trials": null, "date": "2024-12-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 20.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-06-18-27-47--llama33-diff"}, {"id": "aider-edit:e10e80cdc8b709e5", "model": "Grok-2", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 58.6, "first_attempt": 45.9, "tasks": 133, "trials": null, "date": "2024-10-13", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 34.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-13-21-33-42--grok2-whole"}, {"id": "aider-edit:3fc04795c113f097", "model": "llama-3.1-70b-instruct", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "whole", "score": 58.6, "first_attempt": 43.6, "tasks": 133, "trials": null, "date": "2024-07-25", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 7.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-25-08-07-45--fireworks-llama-70b-whole"}, {"id": "aider-edit:6387d56285a780a4", "model": "ollama/qwen2.5:32b-instruct-q8_0", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "whole", "score": 58.6, "first_attempt": 46.6, "tasks": 133, "trials": null, "date": "2024-09-20", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 1763.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-20-21-47-17--qwen2.5-32b-instruct-q8_0-whole"}, {"id": "aider-edit:d9af910cdd529fa6", "model": "Qwen2.5-Coder-7B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 57.9, "first_attempt": 51.9, "tasks": 133, "trials": null, "date": "2024-10-16", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 10.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-16-16-20-59--Qwen2.5-Coder-7B-Instruct"}, {"id": "aider-edit:f8a6203f186dc3c9", "model": "gemini-exp-1121", "agent": "Aider", "agent_version": "0.63.3.dev", "setting": "diff", "score": 57.9, "first_attempt": 46.6, "tasks": 133, "trials": null, "date": "2024-11-21", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 60.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-21-17-46-36--gemini-exp-1121-diff"}, {"id": "aider-edit:02d137d520f5677b", "model": "gpt-3.5-turbo-0301", "agent": "Aider", "agent_version": "0.16.4-dev", "setting": "whole", "score": 57.9, "first_attempt": 50.4, "tasks": 133, "trials": null, "date": "2023-11-06", "cost_usd": 0.4822, "cost_per_trial_usd": null, "seconds": 6.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2023-11-06-21-23-59--gpt-3.5-turbo-0301"}, {"id": "aider-edit:6a015b0526d92577", "model": "gpt-4-turbo-2024-04-09 (diff)", "agent": "Aider", "agent_version": "0.28.1-dev", "setting": "diff", "score": 57.6, "first_attempt": 48.5, "tasks": 33, "trials": null, "date": "2024-04-12", "cost_usd": 1.6205, "cost_per_trial_usd": null, "seconds": 17.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "33", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-04-12-22-18-20--gpt-4-turbo-2024-04-09-plain-diff"}, {"id": "aider-edit:378c5e20b6f4a852", "model": "gemini-1.5-pro-001", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "diff-fenced", "score": 57.1, "first_attempt": 45.9, "tasks": 133, "trials": null, "date": "2024-05-03", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 21.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-03-20-47-24--gemini-1.5-pro-diff-fenced"}, {"id": "aider-edit:37918f03aa9b1910", "model": "gpt-3.5-turbo-1106", "agent": "Aider", "agent_version": "0.30.2-dev", "setting": "whole", "score": 56.1, "first_attempt": 45.5, "tasks": 132, "trials": null, "date": "2024-04-30", "cost_usd": 0.3261, "cost_per_trial_usd": null, "seconds": 5.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "132", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-04-30-21-40-51--litellm-gpt-3.5-turbo-1106-again"}, {"id": "aider-edit:0d3339089cb1c5d6", "model": "Qwen2 72B Instruct", "agent": "Aider", "agent_version": "0.37.1-dev", "setting": "whole", "score": 55.6, "first_attempt": 44.4, "tasks": 133, "trials": null, "date": "2024-06-08", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 14.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-06-08-22-37-55--qwen2-72b-instruct-whole"}, {"id": "aider-edit:702ae140b2d37fcd", "model": "gpt-4o-mini", "agent": "Aider", "agent_version": "0.44.1-dev", "setting": "whole", "score": 55.6, "first_attempt": 40.6, "tasks": 133, "trials": null, "date": "2024-07-18", "cost_usd": 0.0916, "cost_per_trial_usd": null, "seconds": 7.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-18-18-57-46--gpt-4o-mini-whole"}, {"id": "aider-edit:5a64933afdd760e8", "model": "Grok-2-mini", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 54.9, "first_attempt": 40.6, "tasks": 133, "trials": null, "date": "2024-10-13", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 32.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-13-23-58-44--grok2mini-whole"}, {"id": "aider-edit:3d4e6a4d785e508f", "model": "Llama-3.1-Nemotron-70B-Instruct-HF", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 54.9, "first_attempt": 36.8, "tasks": 133, "trials": null, "date": "2024-10-16", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 64.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-16-15-55-37--nemotron-glhf-whole3"}, {"id": "aider-edit:17481e4e46954199", "model": "claude-3-sonnet-20240229", "agent": "Aider", "agent_version": "0.25.1-dev", "setting": "whole", "score": 54.9, "first_attempt": 43.6, "tasks": 133, "trials": null, "date": "2024-03-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 23.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-03-06-16-42-00--claude3-sonnet-whole"}, {"id": "aider-edit:1f86d63c30134f25", "model": "Nova Pro", "agent": "Aider", "agent_version": "0.66.1.dev", "setting": "whole", "score": 54.1, "first_attempt": 44.4, "tasks": 133, "trials": null, "date": "2024-12-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 8.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-04-13-53-03--nova-whole"}, {"id": "aider-edit:5c2abc8e1be59b2f", "model": "Yi Coder 9B Chat", "agent": "Aider", "agent_version": "0.54.13.dev", "setting": "whole", "score": 54.1, "first_attempt": 46.6, "tasks": 133, "trials": null, "date": "2024-09-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 8.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-04-16-08-09--yi-coder-9b-whole"}, {"id": "aider-edit:f30eee50f79d516a", "model": "ollama/qwen2.5:32b", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 54.1, "first_attempt": 44.4, "tasks": 133, "trials": null, "date": "2024-09-30", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 134.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-30-14-09-43--qwen2.5-32b-whole-2"}, {"id": "aider-edit:8df0afb2c4aa6d9b", "model": "gemini-1.5-flash-exp-0827", "agent": "Aider", "agent_version": "0.53.1-dev", "setting": "whole", "score": 52.6, "first_attempt": 40.6, "tasks": 133, "trials": null, "date": "2024-08-27", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 6.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-08-27-19-20-19--gemini-1.5-flash-exp-0827"}, {"id": "aider-edit:c9bc75e839b7cec9", "model": "qwen2.5-coder:7b-instruct-q8_0", "agent": "Aider", "agent_version": "0.56.0", "setting": "whole", "score": 51.9, "first_attempt": 45.1, "tasks": 133, "trials": null, "date": "2024-09-19", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 9.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-19-16-58-29--qwen2.5-coder:7b-instruct-q8_0"}, {"id": "aider-edit:047b7750c44a33c5", "model": "codestral-2405", "agent": "Aider", "agent_version": "0.37.1-dev", "setting": "whole", "score": 51.1, "first_attempt": 35.3, "tasks": 133, "trials": null, "date": "2024-06-09", "cost_usd": 0.6805, "cost_per_trial_usd": null, "seconds": 7.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-06-09-03-28-21--codestral-whole"}, {"id": "aider-edit:7994c9d444feb992", "model": "gemini-1.5-flash-002 (0924)", "agent": "Aider", "agent_version": "0.57.2.dev", "setting": "whole", "score": 51.1, "first_attempt": 37.6, "tasks": 133, "trials": null, "date": "2024-09-24", "cost_usd": 0.0515, "cost_per_trial_usd": null, "seconds": 5.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-24-16-33-23--gemini-1.5-flash-002-whole"}, {"id": "aider-edit:1e6e032ab72a76f9", "model": "gpt-3.5-turbo-0125", "agent": "Aider", "agent_version": "0.33.1-dev", "setting": "whole", "score": 50.4, "first_attempt": 41.4, "tasks": 133, "trials": null, "date": "2024-05-08", "cost_usd": 0.5032, "cost_per_trial_usd": null, "seconds": 6.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-08-20-59-15--may-gpt-3.5-turbo-whole"}, {"id": "aider-edit:f28303cd21b3b839", "model": "gpt-3.5-turbo-0613", "agent": "Aider", "agent_version": "0.16.4-dev", "setting": "whole", "score": 50.4, "first_attempt": 38.3, "tasks": 133, "trials": null, "date": "2023-11-07", "cost_usd": 0.5366, "cost_per_trial_usd": null, "seconds": 18.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2023-11-07-02-41-07--gpt-3.5-turbo-0613"}, {"id": "aider-edit:e12a0a06cf69fd0f", "model": "qwen2:72b-instruct-q8_0", "agent": "Aider", "agent_version": "0.37.1-dev", "setting": "whole", "score": 49.6, "first_attempt": 43.6, "tasks": 133, "trials": null, "date": "2024-06-08", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 280.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-06-08-17-54-04--qwen2:72b-instruct-q8_0-whole"}, {"id": "aider-edit:e2632006e488fb18", "model": "llama3-70b-8192", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "diff", "score": 49.2, "first_attempt": 38.6, "tasks": 132, "trials": null, "date": "2024-05-03", "cost_usd": 0.4311, "cost_per_trial_usd": null, "seconds": 14.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "132", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-03-22-24-48--openrouter--llama3-diff-examples-sys-msg"}, {"id": "aider-edit:60704559c4f55540", "model": "Codestral-22B-v0.1-Q4_K_M", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "whole", "score": 48.1, "first_attempt": 36.1, "tasks": 133, "trials": null, "date": "2024-09-21", "cost_usd": 0.9108, "cost_per_trial_usd": null, "seconds": 656.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-21-11-56-43--Codestral-22B-v0.1-Q4_K_M.gguf_whole"}, {"id": "aider-edit:1a9fa39758146841", "model": "codestral:22b-v0.1-q8_0", "agent": "Aider", "agent_version": "0.37.1-dev", "setting": "whole", "score": 48.1, "first_attempt": 35.3, "tasks": 133, "trials": null, "date": "2024-06-08", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 46.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-06-08-16-12-31--codestral:22b-v0.1-q8_0-whole"}, {"id": "aider-edit:9ef1736b81b48f2b", "model": "claude-3-haiku-20240307", "agent": "Aider", "agent_version": "0.41.1-dev", "setting": "whole", "score": 47.4, "first_attempt": 40.6, "tasks": 133, "trials": null, "date": "2024-07-01", "cost_usd": 0.1946, "cost_per_trial_usd": null, "seconds": 7.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-01-21-41-48--haiku-whole"}, {"id": "aider-edit:31458675ef2d6f21", "model": "ollama/codestral", "agent": "Aider", "agent_version": "0.57.2.dev", "setting": "whole", "score": 45.9, "first_attempt": 33.8, "tasks": 133, "trials": null, "date": "2024-09-28", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 67.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-28-18-30-20--codestral-whole"}, {"id": "aider-edit:c6ca3edc5adbf4db", "model": "yi-coder:9b-chat-q4_0", "agent": "Aider", "agent_version": "0.54.13.dev", "setting": "whole", "score": 45.1, "first_attempt": 41.4, "tasks": 133, "trials": null, "date": "2024-09-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 125.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-04-16-17-33--yi-coder-9b-chat-q4_0-whole"}, {"id": "aider-edit:03099b9f40941950", "model": "WizardLM-2 8x22B", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "whole", "score": 44.4, "first_attempt": 27.8, "tasks": 133, "trials": null, "date": "2024-05-07", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 36.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-07-20-57-04--wizardlm-2-8x22b-whole"}, {"id": "aider-edit:e032d89f30c6bcfe", "model": "gemini-1.5-flash-latest", "agent": "Aider", "agent_version": "0.37.1-dev", "setting": "whole", "score": 44.4, "first_attempt": 33.8, "tasks": 133, "trials": null, "date": "2024-06-08", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 7.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-06-08-23-45-41--gemini-1.5-flash-latest-whole"}, {"id": "aider-edit:4211f7016f1dd169", "model": "ollama/yi-coder:9b-chat-fp16", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 43.6, "first_attempt": 39.8, "tasks": 133, "trials": null, "date": "2024-10-01", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 63.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-01-07-05-40--yi-coder-9b-chat-fp16-whole-1"}, {"id": "aider-edit:d23fd051e65f62db", "model": "Reflection-70B", "agent": "Aider", "agent_version": "0.55.1.dev", "setting": "whole", "score": 42.1, "first_attempt": 33.1, "tasks": 133, "trials": null, "date": "2024-09-06", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 61.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-06-19-55-17--reflection-hyperbolic-whole-output2"}, {"id": "aider-edit:714e4cf15acb51a6", "model": "Qwen2.5-Coder-3B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 39.1, "first_attempt": 33.8, "tasks": 133, "trials": null, "date": "2024-11-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 18.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-04-02-25-32--Qwen2.5-Coder-3B-Instruct"}, {"id": "aider-edit:1bc5c64031cdb9a5", "model": "Command R (08-24)", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "whole", "score": 38.3, "first_attempt": 30.1, "tasks": 133, "trials": null, "date": "2024-09-11", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 7.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-11-15-47-02--command-r-08-2024-whole"}, {"id": "aider-edit:c388755f8d76ff90", "model": "Command R+ (08-24)", "agent": "Aider", "agent_version": "0.56.1.dev", "setting": "whole", "score": 38.3, "first_attempt": 27.1, "tasks": 133, "trials": null, "date": "2024-09-11", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 20.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-11-15-42-17--command-r-plus-08-2024-whole"}, {"id": "aider-edit:cbb5563f42836063", "model": "gemini-1.5-flash-8b-exp-0827", "agent": "Aider", "agent_version": "0.53.1-dev", "setting": "whole", "score": 38.3, "first_attempt": 31.6, "tasks": 133, "trials": null, "date": "2024-08-27", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 7.2, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-08-27-19-42-05--gemini-1.5-flash-8b-exp-0827"}, {"id": "aider-edit:311a9299646774ad", "model": "gemini-1.5-flash-8b-exp-0924", "agent": "Aider", "agent_version": "0.57.2.dev", "setting": "whole", "score": 38.3, "first_attempt": 33.1, "tasks": 133, "trials": null, "date": "2024-09-24", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 6.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-24-15-18-59--gemini-1.5-flash-8b-exp-0924-whole"}, {"id": "aider-edit:a412478dc5cd0dfb", "model": "ollama/mistral-small", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 38.3, "first_attempt": 30.1, "tasks": 133, "trials": null, "date": "2024-10-01", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 84.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-01-02-33-11--mistral-small-whole-1"}, {"id": "aider-edit:50cbd47d93bc2884", "model": "llama-3.1-8b-instruct", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "whole", "score": 37.6, "first_attempt": 26.3, "tasks": 133, "trials": null, "date": "2024-07-25", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 3.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-25-08-12-27--fireworks-llama-8b-whole"}, {"id": "aider-edit:46cf0da57d912eb6", "model": "qwen1.5-110b-chat", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "whole", "score": 37.6, "first_attempt": 30.8, "tasks": 133, "trials": null, "date": "2024-05-07", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 46.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-07-20-32-37--qwen1.5-110b-chat-whole"}, {"id": "aider-edit:15838f7358c373b9", "model": "gemma2:27b-instruct-q8_0", "agent": "Aider", "agent_version": "0.43.0", "setting": "whole", "score": 36.1, "first_attempt": 31.6, "tasks": 133, "trials": null, "date": "2024-07-09", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 101.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-07-09-10-12-27--gemma2:27b-instruct-q8_0"}, {"id": "aider-edit:989bbcb80288c01e", "model": "codeqwen:7b-chat-v1.5-q8_0", "agent": "Aider", "agent_version": "0.37.1-dev", "setting": "whole", "score": 34.6, "first_attempt": 32.3, "tasks": 133, "trials": null, "date": "2024-06-08", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 15.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-06-08-19-25-26--codeqwen:7b-chat-v1.5-q8_0-whole"}, {"id": "aider-edit:7c6c5f7a71e82c1b", "model": "ollama/mistral-nemo:12b-instruct-2407-q4_K_M", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 33.1, "first_attempt": 22.6, "tasks": 133, "trials": null, "date": "2024-09-30", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 34.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-30-03-49-01--mistral-nemo-12b-instruct-2407-q4_K_M-whole-1"}, {"id": "aider-edit:6082ab691dddc1bc", "model": "ollama/codegeex4", "agent": "Aider", "agent_version": "0.57.2.dev", "setting": "whole", "score": 32.3, "first_attempt": 28.6, "tasks": 133, "trials": null, "date": "2024-09-29", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 128.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-29-17-51-11--codegeex4-whole-2"}, {"id": "aider-edit:4da50184c3b4ab61", "model": "Qwen2.5-Coder-1.5B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 31.6, "first_attempt": 28.6, "tasks": 133, "trials": null, "date": "2024-11-11", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 27.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-11-19-37-01--Qwen2.5-Coder-1.5B-Instruct"}, {"id": "aider-edit:7d0d38286ff18ee6", "model": "command-r-plus", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "whole", "score": 31.6, "first_attempt": 21.8, "tasks": 133, "trials": null, "date": "2024-05-06", "cost_usd": 2.7494, "cost_per_trial_usd": null, "seconds": 22.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-05-06-18-31-08--command-r-plus-whole-final"}, {"id": "aider-edit:afd84052f6ef4eba", "model": "ollama/hermes3:8b-llama3.1-fp16", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 30.1, "first_attempt": 24.1, "tasks": 133, "trials": null, "date": "2024-09-30", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 64.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-30-23-01-24--hermes3-8b-llama3.1-fp16-whole-2"}, {"id": "aider-edit:5e96b0cf5b1295bc", "model": "ollama/wojtek/opencodeinterpreter:6.7b", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 30.1, "first_attempt": 26.3, "tasks": 133, "trials": null, "date": "2024-09-30", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 59.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-30-00-09-00--wojtek-opencodeinterpreter-6.7b-whole-2"}, {"id": "aider-edit:b911f84f4e85d6c0", "model": "o1-mini-2024-09-12", "agent": "Aider", "agent_version": "0.69.2.dev", "setting": "whole", "score": 27.1, "first_attempt": 8.9, "tasks": 225, "trials": null, "date": "2024-12-21", "cost_usd": 17.627, "cost_per_trial_usd": null, "seconds": 34.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "225", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-12-21-22-06-01--polyglot-o1-mini-whole"}, {"id": "aider-edit:083819d761c623dd", "model": "ollama/llama3.2:3b-instruct-fp16", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 26.3, "first_attempt": 20.3, "tasks": 133, "trials": null, "date": "2024-09-30", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 66.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-09-30-19-35-40--llama3.2-3b-instruct-fp16-whole-1"}, {"id": "aider-edit:fd4ca95688103940", "model": "ollama/tulu3", "agent": "Aider", "agent_version": "0.65.2.dev", "setting": "whole", "score": 26.3, "first_attempt": 21.8, "tasks": 133, "trials": null, "date": "2024-11-28", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 35.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-28-13-14-00--tulu3-whole-2"}, {"id": "aider-edit:0c6b40a3a6e714ed", "model": "ollama/hermes3", "agent": "Aider", "agent_version": "0.58.1.dev", "setting": "whole", "score": 22.6, "first_attempt": 21.1, "tasks": 133, "trials": null, "date": "2024-10-01", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 24.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-01-16-50-09--hermes3-whole-4"}, {"id": "aider-edit:759beb7dc75dcbd6", "model": "ollama/granite3-dense:8b", "agent": "Aider", "agent_version": "0.65.2.dev", "setting": "whole", "score": 20.3, "first_attempt": 17.3, "tasks": 133, "trials": null, "date": "2024-11-28", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 38.1, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-11-28-14-41-46--granite3-dense-8b-whole-1"}, {"id": "aider-edit:3e4a99c8fad8ee5c", "model": "Qwen2.5-Coder-0.5B-Instruct", "agent": "Aider", "agent_version": "0.59.2.dev", "setting": "whole", "score": 14.3, "first_attempt": 14.3, "tasks": 133, "trials": null, "date": "2024-10-29", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 16.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "133", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/edit_leaderboard.yml", "file": "2024-10-29-00-29-09--Qwen2.5-Coder-0.5B-Instruct"}], "source": "https://github.com/Aider-AI/aider", "revision": "5dc9490bb35f9729ef2c95d00a19ccd30c26339c", "checksum": "4c437c9e22c7fff8b0d61cb8fd4bf9bdf44370783ac9b00a840a80c81319d36e", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", "checked_at": "2026-10-08T08:27:45Z", "updated_at": "2026-10-08T08:27:45Z", "latest_result": "2024-12-21", "status": "ok", "status_message": "Source checked successfully", "cohorts": ["133", "132", "225", "36", "33"]}, {"id": "aider-refactor", "name": "Aider Refactoring", "metric": "First-attempt pass rate", "description": "Historical refactoring runs using Aider. Scores use one attempt, unlike the other Aider suites.", "historical": true, "rows": [{"id": "aider-refactor:53f4909ad301d176", "model": "claude-3-5-sonnet-20241022", "agent": "Aider", "agent_version": "0.60.1.dev", "setting": "diff", "score": 92.1, "first_attempt": 92.1, "tasks": 89, "trials": null, "date": "2024-10-22", "cost_usd": 8.4644, "cost_per_trial_usd": null, "seconds": 32.5, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-10-22-19-57-27--refac-openrouter-sonnet-1022"}, {"id": "aider-refactor:db2c4f85a9708faf", "model": "o1-preview", "agent": "Aider", "agent_version": "0.60.1.dev", "setting": "diff", "score": 75.3, "first_attempt": 75.3, "tasks": 89, "trials": null, "date": "2024-10-22", "cost_usd": 120.985, "cost_per_trial_usd": null, "seconds": 231.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-10-22-20-26-36--refac-o1preview"}, {"id": "aider-refactor:fb9afb79fb5d39a4", "model": "claude-3-opus-20240229", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "diff", "score": 72.3, "first_attempt": 72.3, "tasks": 83, "trials": null, "date": "2024-05-04", "cost_usd": 27.9176, "cost_per_trial_usd": null, "seconds": 67.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "83", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-05-04-17-45-53--refac-opus"}, {"id": "aider-refactor:c05656921762b314", "model": "claude-3.5-sonnet-20240620", "agent": "Aider", "agent_version": "0.40.7-dev", "setting": "diff", "score": 64.0, "first_attempt": 64.0, "tasks": 89, "trials": null, "date": "2024-07-01", "cost_usd": 11.5242, "cost_per_trial_usd": null, "seconds": 42.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-07-01-18-30-33--refac-claude-3.5-sonnet-diff-not-lazy"}, {"id": "aider-refactor:47c215d81a213e17", "model": "gpt-4o", "agent": "Aider", "agent_version": "0.34.1-dev", "setting": "diff", "score": 62.9, "first_attempt": 62.9, "tasks": 89, "trials": null, "date": "2024-05-13", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 27.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-05-13-17-42-22--refac-gpt-4o-diff"}, {"id": "aider-refactor:dfc8e269b788be90", "model": "gpt-4-1106-preview", "agent": "Aider", "agent_version": "0.33.1-dev", "setting": "udiff", "score": 50.6, "first_attempt": 50.6, "tasks": 89, "trials": null, "date": "2024-05-08", "cost_usd": 18.3844, "cost_per_trial_usd": null, "seconds": 61.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-05-08-21-24-16--may-refac-gpt-4-1106-preview"}, {"id": "aider-refactor:2a8c084db6ee920f", "model": "gemini/gemini-1.5-pro-latest", "agent": "Aider", "agent_version": "0.31.2-dev", "setting": "diff-fenced", "score": 49.4, "first_attempt": 49.4, "tasks": 89, "trials": null, "date": "2024-05-04", "cost_usd": null, "cost_per_trial_usd": null, "seconds": 55.7, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-05-04-23-27-02--refac-gemini"}, {"id": "aider-refactor:36238372b922e27b", "model": "gpt-4o-2024-08-06", "agent": "Aider", "agent_version": "0.48.1-dev", "setting": "diff", "score": 49.4, "first_attempt": 49.4, "tasks": 89, "trials": null, "date": "2024-08-06", "cost_usd": 4.0873, "cost_per_trial_usd": null, "seconds": 16.9, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-08-06-18-44-03--refac-gpt-4o-2024-08-06-diff"}, {"id": "aider-refactor:89bcfd5f69c676d0", "model": "o1-mini", "agent": "Aider", "agent_version": "0.60.1.dev", "setting": "diff", "score": 44.9, "first_attempt": 44.9, "tasks": 89, "trials": null, "date": "2024-10-22", "cost_usd": 29.0492, "cost_per_trial_usd": null, "seconds": 115.3, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-10-22-20-03-10--refac-o1mini"}, {"id": "aider-refactor:dd021e739b3427bf", "model": "gpt-4-turbo-2024-04-09 (udiff)", "agent": "Aider", "agent_version": "0.27.1-dev", "setting": "udiff", "score": 34.1, "first_attempt": 34.1, "tasks": 88, "trials": null, "date": "2024-04-09", "cost_usd": 19.6556, "cost_per_trial_usd": null, "seconds": 42.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "88", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-04-09-21-49-54--refac-gpt-4-turbo-2024-04-09"}, {"id": "aider-refactor:56301ac8a185faea", "model": "gpt-4-0125-preview", "agent": "Aider", "agent_version": "0.33.1-dev", "setting": "udiff", "score": 33.7, "first_attempt": 33.7, "tasks": 89, "trials": null, "date": "2024-05-08", "cost_usd": 20.327, "cost_per_trial_usd": null, "seconds": 56.6, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-05-08-22-25-41--may-refac-gpt-4-0125-preview-ex-sys"}, {"id": "aider-refactor:c17297248d56e277", "model": "DeepSeek Coder V2 0724 (deprecated)", "agent": "Aider", "agent_version": "0.45.2-dev", "setting": "diff", "score": 32.6, "first_attempt": 32.6, "tasks": 89, "trials": null, "date": "2024-07-24", "cost_usd": 0.4148, "cost_per_trial_usd": null, "seconds": 85.0, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-07-24-07-49-39--refac-deepseek-coder-v2-0724"}, {"id": "aider-refactor:9315793a085b3097", "model": "DeepSeek Chat V2.5", "agent": "Aider", "agent_version": "0.55.1.dev", "setting": "diff", "score": 31.5, "first_attempt": 31.5, "tasks": 89, "trials": null, "date": "2024-09-05", "cost_usd": 1.0338, "cost_per_trial_usd": null, "seconds": 225.4, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "89", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-09-05-15-19-05--refac-deepseek-v2.5-no-shell"}, {"id": "aider-refactor:48ff7f7082b448ab", "model": "gpt-4-turbo-2024-04-09 (diff)", "agent": "Aider", "agent_version": "0.28.1-dev", "setting": "diff", "score": 21.4, "first_attempt": 21.4, "tasks": 88, "trials": null, "date": "2024-04-10", "cost_usd": 20.4889, "cost_per_trial_usd": null, "seconds": 67.8, "uncertainty": null, "uncertainty_kind": null, "disqualified_trials": null, "cohort": "88", "source": "https://github.com/Aider-AI/aider/blob/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/refactor_leaderboard.yml", "file": "2024-04-10-13-26-18--refac-gpt-4-turbo-2024-04-09-diff"}], "source": "https://github.com/Aider-AI/aider", "revision": "5dc9490bb35f9729ef2c95d00a19ccd30c26339c", "checksum": "e01b88adf869b88501d48f9b041dbbb37e77e464e1bed886bb62f6b335c60561", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", "checked_at": "2026-10-08T08:27:45Z", "updated_at": "2026-10-08T08:27:45Z", "latest_result": "2024-10-22", "status": "ok", "status_message": "Source checked successfully", "cohorts": ["89", "88", "83"]}]}