[ { "space": "healthbench-consensus-medgemma", "file": "2026-08-05T08-51-22-00-00_healthbench-consensus_6a8D2h492in82NNQHCFgZw.eval", "task": "healthbench_consensus", "task_version": 2, "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "judge_model": "openai/gpt-4.1", "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-05T08:51:22+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.9105, "_scorer/std": 0.0032, "_scorer/criteria_met_rate": 0.9066, "healthbench_score/mean": 0.9105, "healthbench_score_length_adjusted/mean": 0.9007 }, "model_usage_models": [ "vllm/google/medgemma-27b-text-it", "openai/gpt-4.1" ], "model_usage_total": { "vllm/google/medgemma-27b-text-it": 3662191, "openai/gpt-4.1": 14399926 }, "error": null, "solver": "generate" }, { "space": "healthbench-consensus-medgemma", "file": "2026-08-05T10-32-09-00-00_healthbench-consensus_6WXSuGaZvDNS5Zze6Jrm3h.eval", "task": "healthbench_consensus", "task_version": 2, "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "judge_model": "openai/gpt-4.1", "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-05T10:32:09+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.7583, "_scorer/std": 0.0055, "_scorer/criteria_met_rate": 0.7605, "healthbench_score/mean": 0.7583, "healthbench_score_length_adjusted/mean": 0.7522 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-4.1" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 2959598, "openai/gpt-4.1": 12979748 }, "error": null, "solver": "generate" }, { "space": "healthbench-hard-medgemma-v2", "file": "2026-08-05T07-54-55-00-00_healthbench-hard_MANxy36rpJhotpAvdwQiFk.eval", "task": "healthbench_hard", "task_version": 2, "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "judge_model": "openai/gpt-4.1", "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-05T07:54:55+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0, "_scorer/criteria_met_rate": 0.3527, "healthbench_score/mean": -0.0347, "healthbench_score_length_adjusted/mean": -0.1256 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-4.1" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 887308, "openai/gpt-4.1": 19808558 }, "error": null, "solver": "generate" }, { "space": "healthbench-hard-medgemma-v2", "file": "2026-08-05T08-07-41-00-00_healthbench-hard_D9KnFzsoQRE57vVsejUQnW.eval", "task": "healthbench_hard", "task_version": 2, "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "judge_model": "openai/gpt-4.1", "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-05T08:07:41+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.1415, "_scorer/std": 0.0099, "_scorer/criteria_met_rate": 0.4468, "healthbench_score/mean": 0.1415, "healthbench_score_length_adjusted/mean": -0.023 }, "model_usage_models": [ "vllm/google/medgemma-27b-text-it", "openai/gpt-4.1" ], "model_usage_total": { "vllm/google/medgemma-27b-text-it": 1130260, "openai/gpt-4.1": 22994024 }, "error": null, "solver": "generate" }, { "space": "healthbench-professional-deepseek-v4-pro", "file": "2026-08-06T02-35-30-00-00_healthbench-professional_bbeHCEMNSj6Q362BQHa7mS.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "retry_on_error": 3, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-06T02:35:30+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": null, "completed_samples": 525, "total_samples": 525, "metrics": { "_scorer/bootstrap_score": 0.3099, "_scorer/std": 0.0279, "_scorer/criteria_met_rate": 0.5921, "_scorer/use_case_consult_score": 0.4605, "_scorer/use_case_consult_std": 0.0305, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.666, "_scorer/use_case_research_std": 0.0324, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.3543, "_scorer/use_case_writing_std": 0.036, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.6265, "_scorer/type_good_faith_std": 0.0226, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.2494, "_scorer/type_red_teaming_std": 0.0298, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.7023, "_scorer/difficulty_typical_std": 0.0225, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.2867, "_scorer/difficulty_difficult_std": 0.0256, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4631, "_scorer/specialty_ent_std": 0.1645, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.338, "_scorer/specialty_gi_std": 0.1003, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.3996, "_scorer/specialty_id_std": 0.1037, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.58, "_scorer/specialty_anesthesia_std": 0.0903, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4885, "_scorer/specialty_cards_std": 0.0681, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.4581, "_scorer/specialty_derm_std": 0.0919, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4193, "_scorer/specialty_emergency_std": 0.1189, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5738, "_scorer/specialty_endo_std": 0.0944, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.875, "_scorer/specialty_genetics_std": 0.1125, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.3914, "_scorer/specialty_heme/onc_std": 0.085, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.3012, "_scorer/specialty_medicine_std": 0.1004, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.6434, "_scorer/specialty_nephro_std": 0.0823, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.6737, "_scorer/specialty_neuro_std": 0.0591, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.5385, "_scorer/specialty_neurosurg_std": 0.1299, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4391, "_scorer/specialty_obgyn_std": 0.0627, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.5012, "_scorer/specialty_ophtho_std": 0.1049, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.4772, "_scorer/specialty_ortho_std": 0.0912, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.4223, "_scorer/specialty_other_std": 0.1018, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.4566, "_scorer/specialty_peds_std": 0.0911, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.4501, "_scorer/specialty_primary_std": 0.16, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.594, "_scorer/specialty_psychiatry_std": 0.0852, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.2632, "_scorer/specialty_pulm_std": 0.1066, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.4375, "_scorer/specialty_rads_std": 0.1592, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.375, "_scorer/specialty_rheum_std": 0.1675, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.6471, "_scorer/specialty_surgery_std": 0.1192, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.5619, "_scorer/specialty_uro_std": 0.1296, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.7023, "_scorer/source_slice_good_faith_typical_std": 0.0222, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.3778, "_scorer/source_slice_good_faith_difficult_std": 0.0544, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2494, "_scorer/source_slice_red_teaming_difficult_std": 0.0297, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.3781, "healthbench_professional_score_length_adjusted/mean": 0.3099 }, "model_usage_models": [ "openrouter/deepseek/deepseek-v4-pro", "openai/gpt-5.4" ], "model_usage_total": { "openrouter/deepseek/deepseek-v4-pro": 1258221, "openai/gpt-5.4": 2810449 }, "error": null, "solver": "generate" }, { "space": "hs-consensus", "file": "2026-07-16T20-09-58-00-00_healthbench-consensus_87a5hkWgWY5rrHyoygzDVx.eval", "task": "inspect_evals/healthbench_consensus", "task_version": 2, "task_file": null, "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T20:09:58+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.7525, "_scorer/std": 0.0052, "_scorer/criteria_met_rate": 0.7369, "healthbench_score/mean": 0.7525, "healthbench_score_length_adjusted/mean": 0.7478 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 12531224 }, "error": null, "solver": "generate" }, { "space": "hs-consensus", "file": "2026-07-16T20-20-37-00-00_healthbench-consensus_gmfrK88PFGpuvRWqnk7uc5.eval", "task": "inspect_evals/healthbench_consensus", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T20:20:37+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.8208, "_scorer/std": 0.0046, "_scorer/criteria_met_rate": 0.8079, "healthbench_score/mean": 0.8208, "healthbench_score_length_adjusted/mean": 0.8202 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 10709027 }, "error": null, "solver": "generate" }, { "space": "hs-consensus", "file": "2026-07-16T20-44-07-00-00_healthbench-consensus_EbuNesZCwbaRVAqRedBwjP.eval", "task": "inspect_evals/healthbench_consensus", "task_version": 2, "task_file": null, "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T20:44:07+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.791, "_scorer/std": 0.0049, "_scorer/criteria_met_rate": 0.7765, "healthbench_score/mean": 0.791, "healthbench_score_length_adjusted/mean": 0.7846 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 13025085 }, "error": null, "solver": "generate" }, { "space": "hs-consensus", "file": "2026-07-16T21-11-43-00-00_healthbench-consensus_k2nEohAktRtjKfZ5BNGJSG.eval", "task": "inspect_evals/healthbench_consensus", "task_version": 2, "task_file": null, "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T21:11:43+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.8017, "_scorer/std": 0.0049, "_scorer/criteria_met_rate": 0.7954, "healthbench_score/mean": 0.8017, "healthbench_score_length_adjusted/mean": 0.8023 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 10263280 }, "error": null, "solver": "generate" }, { "space": "hs-consensus", "file": "2026-07-24T09-07-55-00-00_healthbench-consensus_QX6zAfF9oDZDAX4p6k6yb9.eval", "task": "inspect_evals/healthbench_consensus", "task_version": 2, "task_file": null, "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T09:07:55+00:00", "inspect_version": { "inspect_ai": "0.3.249", "inspect_evals": "0.16.0" }, "revision": null, "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.7137, "_scorer/std": 0.006, "_scorer/criteria_met_rate": 0.7011, "healthbench_score/mean": 0.7137, "healthbench_score_length_adjusted/mean": 0.7077 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-4o-mini" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 3440824, "openai/gpt-4o-mini": 14762331 }, "error": null, "solver": "generate" }, { "space": "hs-consensus", "file": "2026-07-24T11-39-48-00-00_healthbench-consensus_7b4ZMKGYfFkgFNSJXJs53z.eval", "task": "inspect_evals/healthbench_consensus", "task_version": 2, "task_file": null, "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.002 }, "dataset_samples": 3671, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 183, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T11:39:48+00:00", "inspect_version": { "inspect_ai": "0.3.249", "inspect_evals": "0.16.0" }, "revision": null, "completed_samples": 3671, "total_samples": 3671, "metrics": { "_scorer/bootstrap_score": 0.7758, "_scorer/std": 0.0049, "_scorer/criteria_met_rate": 0.7616, "healthbench_score/mean": 0.7758, "healthbench_score_length_adjusted/mean": 0.7661 }, "model_usage_models": [ "vllm/google/medgemma-27b-text-it", "openai/gpt-4o-mini" ], "model_usage_total": { "vllm/google/medgemma-27b-text-it": 4303064, "openai/gpt-4o-mini": 16607219 }, "error": null, "solver": "generate" }, { "space": "hs-hard", "file": "2026-07-16T07-28-45-00-00_healthbench-hard_dxTyk6Wmosx3kJAp4iULs9.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T07:28:45+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.2727, "_scorer/std": 0.0099, "_scorer/criteria_met_rate": 0.3811, "healthbench_score/mean": 0.2727, "healthbench_score_length_adjusted/mean": 0.2603 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 17512569 }, "error": null, "solver": "generate" }, { "space": "hs-hard", "file": "2026-07-16T18-52-11-00-00_healthbench-hard_Z9QDknMd4SukiAP7Q4FSPz.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T18:52:11+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.2661, "_scorer/std": 0.0095, "_scorer/criteria_met_rate": 0.3611, "healthbench_score/mean": 0.2661, "healthbench_score_length_adjusted/mean": 0.278 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 16665282 }, "error": null, "solver": "generate" }, { "space": "hs-hard", "file": "2026-07-16T19-07-31-00-00_healthbench-hard_SABwaQjc3Xon5iDMpWFQV3.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T19:07:31+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.2484, "_scorer/std": 0.0096, "_scorer/criteria_met_rate": 0.354, "healthbench_score/mean": 0.2484, "healthbench_score_length_adjusted/mean": 0.138 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 21037790 }, "error": null, "solver": "generate" }, { "space": "hs-hard", "file": "2026-07-16T19-23-40-00-00_healthbench-hard_mEs7dJHDapQ4JVqHQtfTJc.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T19:23:40+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.1743, "_scorer/std": 0.0084, "_scorer/criteria_met_rate": 0.2732, "healthbench_score/mean": 0.1743, "healthbench_score_length_adjusted/mean": 0.0961 }, "model_usage_models": [ "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-4o-mini": 20067380 }, "error": null, "solver": "generate" }, { "space": "hs-hard", "file": "2026-07-24T09-18-15-00-00_healthbench-hard_WkGg4UV64qpccys2fbViHY.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T09:18:15+00:00", "inspect_version": { "inspect_ai": "0.3.249", "inspect_evals": "0.16.0" }, "revision": null, "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.106, "_scorer/std": 0.0077, "_scorer/criteria_met_rate": 0.2222, "healthbench_score/mean": 0.106, "healthbench_score_length_adjusted/mean": 0.0126 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-4o-mini" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 402479, "openai/gpt-4o-mini": 17583141 }, "error": null, "solver": "generate" }, { "space": "hs-hard", "file": "2026-07-24T14-36-42-00-00_healthbench-hard_BU7wDz79JMGuziKwJrRrpN.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T14:36:42+00:00", "inspect_version": { "inspect_ai": "0.3.249", "inspect_evals": "0.16.0" }, "revision": null, "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.2109, "_scorer/std": 0.0083, "_scorer/criteria_met_rate": 0.3143, "healthbench_score/mean": 0.2109, "healthbench_score_length_adjusted/mean": 0.0483 }, "model_usage_models": [ "openai/gpt-4o-mini", "vllm/google/medgemma-27b-text-it" ], "model_usage_total": { "openai/gpt-4o-mini": 20040174, "vllm/google/medgemma-27b-text-it": 24135 }, "error": null, "solver": "generate" }, { "space": "hs-non-professional", "file": "2026-07-15T06-51-58-00-00_healthbench_WJDhL9LgZJKMGgYizSLYpL.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4.1", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0299 }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-15T06:51:58+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.5687, "_scorer/std": 0.0048, "_scorer/criteria_met_rate": 0.5365, "healthbench_score/mean": 0.5687, "healthbench_score_length_adjusted/mean": 0.5583 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-non-professional", "file": "2026-07-15T07-01-24-00-00_healthbench_Hah9CRDdSrTz7K5EFgRWF8.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4.1", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0299 }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-15T07:01:24+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.5344, "_scorer/std": 0.0048, "_scorer/criteria_met_rate": 0.5323, "healthbench_score/mean": 0.5344, "healthbench_score_length_adjusted/mean": 0.5431 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-non-professional", "file": "2026-07-16T04-29-43-00-00_healthbench_jBnP8VSE7rSvrMpzRStQFK.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4.1", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0299 }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T04:29:43+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.5137, "_scorer/std": 0.0051, "_scorer/criteria_met_rate": 0.542, "healthbench_score/mean": 0.5137, "healthbench_score_length_adjusted/mean": 0.4173 }, "model_usage_models": [ "openrouter/deepseek/deepseek-v4-pro", "openai/gpt-4.1" ], "model_usage_total": { "openrouter/deepseek/deepseek-v4-pro": 7415173, "openai/gpt-4.1": 105583667 }, "error": null, "solver": "generate" }, { "space": "hs-non-professional", "file": "2026-07-16T05-23-21-00-00_healthbench_iexipStaaxm8E5ni3Gg96Z.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4.1", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0299 }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-16T05:23:21+00:00", "inspect_version": { "inspect_ai": "0.3.246", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.3942, "_scorer/std": 0.0054, "_scorer/criteria_met_rate": 0.4698, "healthbench_score/mean": 0.3942, "healthbench_score_length_adjusted/mean": 0.3243 }, "model_usage_models": [ "openai/gpt-4.1", "openai-api/plamo/plamo-3.0-prime" ], "model_usage_total": { "openai/gpt-4.1": 49156082, "openai-api/plamo/plamo-3.0-prime": 2449482 }, "error": null, "solver": "generate" }, { "space": "hs-non-professional", "file": "2026-07-24T09-32-53-00-00_healthbench_6Co8iMuXsKxGNrMeAkRbdp.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4.1", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0299 }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T09:32:53+00:00", "inspect_version": { "inspect_ai": "0.3.249", "inspect_evals": "0.16.0" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.2697, "_scorer/std": 0.0057, "_scorer/criteria_met_rate": 0.4107, "healthbench_score/mean": 0.2697, "healthbench_score_length_adjusted/mean": 0.1832 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-4.1" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 877814, "openai/gpt-4.1": 100167550 }, "error": null, "solver": "generate" }, { "space": "hs-non-professional", "file": "2026-07-24T12-08-49-00-00_healthbench_cuBWJL4bbU32vR5cxHsjEC.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4.1", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0299 }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T12:08:49+00:00", "inspect_version": { "inspect_ai": "0.3.249", "inspect_evals": "0.16.0" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.472, "_scorer/std": 0.0054, "_scorer/criteria_met_rate": 0.5262, "healthbench_score/mean": 0.472, "healthbench_score_length_adjusted/mean": 0.3321 }, "model_usage_models": [ "openai/gpt-4.1", "vllm/google/medgemma-27b-text-it" ], "model_usage_total": { "openai/gpt-4.1": 113164489, "vllm/google/medgemma-27b-text-it": 1556281 }, "error": null, "solver": "generate" }, { "space": "hs-original", "file": "2026-07-24T15-57-49-00-00_healthbench-professional_7PZFsBJYTZRCdXZkJPfki7.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000.0, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T15:57:49+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4781, "_scorer/std": 0.0226, "_scorer/criteria_met_rate": 0.635, "_scorer/use_case_consult_score": 0.6058, "_scorer/use_case_consult_std": 0.0292, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.7055, "_scorer/use_case_research_std": 0.0308, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.5245, "_scorer/use_case_writing_std": 0.0379, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.6996, "_scorer/type_good_faith_std": 0.0213, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.458, "_scorer/type_red_teaming_std": 0.0339, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.757, "_scorer/difficulty_typical_std": 0.0215, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.4735, "_scorer/difficulty_difficult_std": 0.0284, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.7583, "_scorer/specialty_ent_std": 0.1338, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.4199, "_scorer/specialty_gi_std": 0.093, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.513, "_scorer/specialty_id_std": 0.0837, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.691, "_scorer/specialty_anesthesia_std": 0.0873, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.5371, "_scorer/specialty_cards_std": 0.0706, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.5994, "_scorer/specialty_derm_std": 0.0884, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.6375, "_scorer/specialty_emergency_std": 0.101, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.7583, "_scorer/specialty_endo_std": 0.0876, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.75, "_scorer/specialty_genetics_std": 0.2205, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.5129, "_scorer/specialty_heme/onc_std": 0.0819, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.4342, "_scorer/specialty_medicine_std": 0.113, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.8512, "_scorer/specialty_nephro_std": 0.0663, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.7139, "_scorer/specialty_neuro_std": 0.0533, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.4606, "_scorer/specialty_neurosurg_std": 0.1454, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.6554, "_scorer/specialty_obgyn_std": 0.0635, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.6142, "_scorer/specialty_ophtho_std": 0.1078, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.4436, "_scorer/specialty_ortho_std": 0.0902, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.4854, "_scorer/specialty_other_std": 0.1011, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.3333, "_scorer/specialty_path_std": 0.2852, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.536, "_scorer/specialty_peds_std": 0.0836, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.7815, "_scorer/specialty_primary_std": 0.1417, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.678, "_scorer/specialty_psychiatry_std": 0.0807, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.7483, "_scorer/specialty_pulm_std": 0.0939, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.6875, "_scorer/specialty_rads_std": 0.1503, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.5, "_scorer/specialty_rheum_std": 0.1817, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.7353, "_scorer/specialty_surgery_std": 0.1033, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6538, "_scorer/specialty_uro_std": 0.1219, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.757, "_scorer/source_slice_good_faith_typical_std": 0.0211, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.5114, "_scorer/source_slice_good_faith_difficult_std": 0.0527, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.458, "_scorer/source_slice_red_teaming_difficult_std": 0.0352, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.5295, "healthbench_professional_score_length_adjusted/mean": 0.4781 }, "model_usage_models": [ "openai/gpt-5.5", "openai/gpt-5.4" ], "model_usage_total": { "openai/gpt-5.5": 8994839, "openai/gpt-5.4": 21393751 }, "error": null, "solver": "generate" }, { "space": "hs-original", "file": "2026-07-24T16-55-39-00-00_healthbench-professional_boiooCVNG7ZkBVsnEpKLNT.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000.0, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T16:55:39+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4803, "_scorer/std": 0.0235, "_scorer/criteria_met_rate": 0.6238, "_scorer/use_case_consult_score": 0.6051, "_scorer/use_case_consult_std": 0.028, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.6739, "_scorer/use_case_research_std": 0.0339, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.5091, "_scorer/use_case_writing_std": 0.037, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.6785, "_scorer/type_good_faith_std": 0.0212, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4583, "_scorer/type_red_teaming_std": 0.0335, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.7291, "_scorer/difficulty_typical_std": 0.0219, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.474, "_scorer/difficulty_difficult_std": 0.03, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.6821, "_scorer/specialty_ent_std": 0.1682, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.4119, "_scorer/specialty_gi_std": 0.0897, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.6207, "_scorer/specialty_id_std": 0.0845, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 0.5, "_scorer/specialty_allergy_std": 0.3595, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.7658, "_scorer/specialty_anesthesia_std": 0.0746, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.5808, "_scorer/specialty_cards_std": 0.0634, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6161, "_scorer/specialty_derm_std": 0.0883, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.5294, "_scorer/specialty_emergency_std": 0.1183, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.6903, "_scorer/specialty_endo_std": 0.0944, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.4205, "_scorer/specialty_genetics_std": 0.2161, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.4822, "_scorer/specialty_heme/onc_std": 0.0857, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.5676, "_scorer/specialty_medicine_std": 0.106, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.7864, "_scorer/specialty_nephro_std": 0.0689, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.7189, "_scorer/specialty_neuro_std": 0.0531, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.5109, "_scorer/specialty_neurosurg_std": 0.1316, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.5429, "_scorer/specialty_obgyn_std": 0.0685, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.5586, "_scorer/specialty_ophtho_std": 0.1146, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.5932, "_scorer/specialty_ortho_std": 0.0957, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.5797, "_scorer/specialty_other_std": 0.0941, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.1905, "_scorer/specialty_path_std": 0.1571, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.4692, "_scorer/specialty_peds_std": 0.0858, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.6387, "_scorer/specialty_primary_std": 0.1686, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.6688, "_scorer/specialty_psychiatry_std": 0.0784, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5967, "_scorer/specialty_pulm_std": 0.1183, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.7463, "_scorer/specialty_rads_std": 0.1261, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.5, "_scorer/specialty_rheum_std": 0.17, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.6765, "_scorer/specialty_surgery_std": 0.1096, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.511, "_scorer/specialty_uro_std": 0.1318, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.7291, "_scorer/source_slice_good_faith_typical_std": 0.0206, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.5125, "_scorer/source_slice_good_faith_difficult_std": 0.0534, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4583, "_scorer/source_slice_red_teaming_difficult_std": 0.034, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.5083, "healthbench_professional_score_length_adjusted/mean": 0.4803 }, "model_usage_models": [ "anthropic/claude-opus-4-7", "openai/gpt-5.4" ], "model_usage_total": { "anthropic/claude-opus-4-7": 9075443, "openai/gpt-5.4": 19817644 }, "error": null, "solver": "generate" }, { "space": "hs-original", "file": "2026-07-24T17-30-37-00-00_healthbench-professional_c9RnXrEaoi4T33bvwXEA5h.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000.0, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T17:30:37+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.1374, "_scorer/std": 0.0236, "_scorer/criteria_met_rate": 0.4202, "_scorer/use_case_consult_score": 0.3313, "_scorer/use_case_consult_std": 0.026, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.4886, "_scorer/use_case_research_std": 0.0334, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.1977, "_scorer/use_case_writing_std": 0.0285, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.4366, "_scorer/type_good_faith_std": 0.0231, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.1688, "_scorer/type_red_teaming_std": 0.0255, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.4965, "_scorer/difficulty_typical_std": 0.0243, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.1894, "_scorer/difficulty_difficult_std": 0.0225, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.1744, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.1772, "_scorer/specialty_gi_std": 0.0766, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.2871, "_scorer/specialty_id_std": 0.0833, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.3928, "_scorer/specialty_anesthesia_std": 0.0903, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4052, "_scorer/specialty_cards_std": 0.0608, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.2958, "_scorer/specialty_derm_std": 0.0803, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4044, "_scorer/specialty_emergency_std": 0.1205, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5108, "_scorer/specialty_endo_std": 0.0907, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.75, "_scorer/specialty_genetics_std": 0.2168, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.2388, "_scorer/specialty_heme/onc_std": 0.0755, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.2962, "_scorer/specialty_medicine_std": 0.1038, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4423, "_scorer/specialty_nephro_std": 0.0844, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.4003, "_scorer/specialty_neuro_std": 0.062, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.4342, "_scorer/specialty_neurosurg_std": 0.1386, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.3389, "_scorer/specialty_obgyn_std": 0.0689, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3904, "_scorer/specialty_ophtho_std": 0.1091, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.3319, "_scorer/specialty_ortho_std": 0.0895, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.2228, "_scorer/specialty_other_std": 0.0781, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.3333, "_scorer/specialty_path_std": 0.272, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.2704, "_scorer/specialty_peds_std": 0.0761, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.4958, "_scorer/specialty_primary_std": 0.1766, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.2619, "_scorer/specialty_psychiatry_std": 0.0695, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.2189, "_scorer/specialty_pulm_std": 0.0956, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.1875, "_scorer/specialty_rads_std": 0.1278, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.3601, "_scorer/specialty_rheum_std": 0.1394, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.3216, "_scorer/specialty_surgery_std": 0.1044, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.3156, "_scorer/specialty_uro_std": 0.0998, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.4965, "_scorer/source_slice_good_faith_typical_std": 0.0249, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.2399, "_scorer/source_slice_good_faith_difficult_std": 0.046, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.1688, "_scorer/source_slice_red_teaming_difficult_std": 0.0264, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.2078, "healthbench_professional_score_length_adjusted/mean": 0.1374 }, "model_usage_models": [ "openai-api/plamo/plamo-3.0-prime", "openai/gpt-5.4" ], "model_usage_total": { "openai-api/plamo/plamo-3.0-prime": 8130263, "openai/gpt-5.4": 23382174 }, "error": null, "solver": "generate" }, { "space": "hs-original", "file": "2026-07-24T22-44-03-00-00_healthbench-professional_6AYXiEDgGuzNtAcyWPDGrR.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000.0, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T22:44:03+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.2003, "_scorer/std": 0.0242, "_scorer/criteria_met_rate": 0.4928, "_scorer/use_case_consult_score": 0.3894, "_scorer/use_case_consult_std": 0.0283, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.5213, "_scorer/use_case_research_std": 0.0327, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.3879, "_scorer/use_case_writing_std": 0.0375, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5249, "_scorer/type_good_faith_std": 0.0231, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.2529, "_scorer/type_red_teaming_std": 0.0283, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5851, "_scorer/difficulty_typical_std": 0.0235, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.2745, "_scorer/difficulty_difficult_std": 0.0263, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.6155, "_scorer/specialty_ent_std": 0.1543, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.1627, "_scorer/specialty_gi_std": 0.0714, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.426, "_scorer/specialty_id_std": 0.0819, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.4063, "_scorer/specialty_anesthesia_std": 0.0882, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4054, "_scorer/specialty_cards_std": 0.0595, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.471, "_scorer/specialty_derm_std": 0.0919, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.5021, "_scorer/specialty_emergency_std": 0.1173, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5296, "_scorer/specialty_endo_std": 0.0885, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.75, "_scorer/specialty_genetics_std": 0.2164, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.3479, "_scorer/specialty_heme/onc_std": 0.0773, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.3804, "_scorer/specialty_medicine_std": 0.115, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.5785, "_scorer/specialty_nephro_std": 0.0882, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.4748, "_scorer/specialty_neuro_std": 0.0612, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.5073, "_scorer/specialty_neurosurg_std": 0.1282, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.403, "_scorer/specialty_obgyn_std": 0.0673, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.4915, "_scorer/specialty_ophtho_std": 0.1119, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.481, "_scorer/specialty_ortho_std": 0.0913, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.388, "_scorer/specialty_other_std": 0.1, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.3969, "_scorer/specialty_peds_std": 0.0854, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1527, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.4225, "_scorer/specialty_psychiatry_std": 0.0769, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.3023, "_scorer/specialty_pulm_std": 0.105, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.375, "_scorer/specialty_rads_std": 0.1699, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.3542, "_scorer/specialty_rheum_std": 0.1343, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.4098, "_scorer/specialty_surgery_std": 0.1092, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.4106, "_scorer/specialty_uro_std": 0.1189, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5851, "_scorer/source_slice_good_faith_typical_std": 0.0242, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.3273, "_scorer/source_slice_good_faith_difficult_std": 0.0509, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2529, "_scorer/source_slice_red_teaming_difficult_std": 0.0283, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.3125, "healthbench_professional_score_length_adjusted/mean": 0.2003 }, "model_usage_models": [ "openai/gpt-5.4", "vllm/google/medgemma-27b-text-it" ], "model_usage_total": { "openai/gpt-5.4": 25383913, "vllm/google/medgemma-27b-text-it": 8383935 }, "error": null, "solver": "generate" }, { "space": "hs-original", "file": "2026-07-25T00-26-29-00-00_healthbench-professional_GEViyNR2PM2B2oUV94DzoK.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000.0, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T00:26:29+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.0904, "_scorer/std": 0.0218, "_scorer/criteria_met_rate": 0.353, "_scorer/use_case_consult_score": 0.2765, "_scorer/use_case_consult_std": 0.0255, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.378, "_scorer/use_case_research_std": 0.0331, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.2347, "_scorer/use_case_writing_std": 0.0317, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.3664, "_scorer/type_good_faith_std": 0.0218, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.1664, "_scorer/type_red_teaming_std": 0.0262, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.4352, "_scorer/difficulty_typical_std": 0.0251, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.1589, "_scorer/difficulty_difficult_std": 0.0205, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4381, "_scorer/specialty_ent_std": 0.1637, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.1968, "_scorer/specialty_gi_std": 0.076, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.2115, "_scorer/specialty_id_std": 0.0761, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.2753, "_scorer/specialty_anesthesia_std": 0.0799, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.326, "_scorer/specialty_cards_std": 0.061, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.3973, "_scorer/specialty_derm_std": 0.085, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4982, "_scorer/specialty_emergency_std": 0.117, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.4903, "_scorer/specialty_endo_std": 0.0787, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.6705, "_scorer/specialty_genetics_std": 0.1998, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.1647, "_scorer/specialty_heme/onc_std": 0.0648, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.1889, "_scorer/specialty_medicine_std": 0.0717, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.3844, "_scorer/specialty_nephro_std": 0.0833, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.3356, "_scorer/specialty_neuro_std": 0.0542, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.3848, "_scorer/specialty_neurosurg_std": 0.1239, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.2405, "_scorer/specialty_obgyn_std": 0.0597, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3773, "_scorer/specialty_ophtho_std": 0.1088, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.2661, "_scorer/specialty_ortho_std": 0.0772, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3561, "_scorer/specialty_other_std": 0.0932, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.2501, "_scorer/specialty_peds_std": 0.0711, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.3745, "_scorer/specialty_primary_std": 0.1358, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.1877, "_scorer/specialty_psychiatry_std": 0.0602, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.148, "_scorer/specialty_pulm_std": 0.0831, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.0625, "_scorer/specialty_rads_std": 0.0575, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.2917, "_scorer/specialty_rheum_std": 0.1439, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.3235, "_scorer/specialty_surgery_std": 0.1087, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.214, "_scorer/specialty_uro_std": 0.086, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.4352, "_scorer/source_slice_good_faith_typical_std": 0.0245, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.1407, "_scorer/source_slice_good_faith_difficult_std": 0.0342, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.1664, "_scorer/source_slice_red_teaming_difficult_std": 0.0245, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.1651, "healthbench_professional_score_length_adjusted/mean": 0.0904 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-5.4" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 7090153, "openai/gpt-5.4": 22550200 }, "error": null, "solver": "generate" }, { "space": "hs-original", "file": "2026-07-25T18-24-55-00-00_healthbench-professional_HaKmMaK4Q6JCLuX2RLCPnc.eval", "task": "healthbench_professional", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "local_path": null, "hf": true, "use_case": null, "usage_type": null, "difficulty": null, "specialty": null, "physician_baseline": false, "judge_model": "openai/gpt-5.4", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "judge_reasoning_effort": "low", "bootstrap_samples": 1000, "include_subset_scores": true, "length_adjustment_center": 2000.0, "length_adjustment_penalty_per_500_chars": 0.0147, "cache": true }, "dataset_samples": 525, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": 0.1, "continue_on_fail": false, "retry_on_error": 5, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T18:24:55+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 525, "total_samples": 525, "metrics": { "_scorer/bootstrap_score": 0.2742, "_scorer/std": 0.0264, "_scorer/criteria_met_rate": 0.5921, "_scorer/use_case_consult_score": 0.4169, "_scorer/use_case_consult_std": 0.0279, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.6383, "_scorer/use_case_research_std": 0.0321, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.3413, "_scorer/use_case_writing_std": 0.0367, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5995, "_scorer/type_good_faith_std": 0.0219, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.2117, "_scorer/type_red_teaming_std": 0.0289, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.6716, "_scorer/difficulty_typical_std": 0.0233, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.2555, "_scorer/difficulty_difficult_std": 0.0245, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.6821, "_scorer/specialty_ent_std": 0.1635, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.2094, "_scorer/specialty_gi_std": 0.0862, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.403, "_scorer/specialty_id_std": 0.0984, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6112, "_scorer/specialty_anesthesia_std": 0.0934, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4409, "_scorer/specialty_cards_std": 0.066, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.4885, "_scorer/specialty_derm_std": 0.0898, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.3237, "_scorer/specialty_emergency_std": 0.1081, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5544, "_scorer/specialty_endo_std": 0.0905, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.5455, "_scorer/specialty_genetics_std": 0.18, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.3489, "_scorer/specialty_heme/onc_std": 0.0825, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.2598, "_scorer/specialty_medicine_std": 0.0969, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.5412, "_scorer/specialty_nephro_std": 0.0855, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5966, "_scorer/specialty_neuro_std": 0.0597, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.4961, "_scorer/specialty_neurosurg_std": 0.1267, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4996, "_scorer/specialty_obgyn_std": 0.0638, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.5038, "_scorer/specialty_ophtho_std": 0.1129, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.4117, "_scorer/specialty_ortho_std": 0.0888, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.4047, "_scorer/specialty_other_std": 0.0994, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.4606, "_scorer/specialty_peds_std": 0.0945, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.4501, "_scorer/specialty_primary_std": 0.159, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5628, "_scorer/specialty_psychiatry_std": 0.0791, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.3439, "_scorer/specialty_pulm_std": 0.1118, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.3708, "_scorer/specialty_rads_std": 0.1446, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.4524, "_scorer/specialty_rheum_std": 0.164, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.5294, "_scorer/specialty_surgery_std": 0.1219, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.3721, "_scorer/specialty_uro_std": 0.1228, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.6716, "_scorer/source_slice_good_faith_typical_std": 0.0218, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.3629, "_scorer/source_slice_good_faith_difficult_std": 0.049, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2117, "_scorer/source_slice_red_teaming_difficult_std": 0.0269, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.343, "healthbench_professional_score_length_adjusted/mean": 0.2742 }, "model_usage_models": [ "openrouter/deepseek/deepseek-v4-pro", "openai/gpt-5.4" ], "model_usage_total": { "openrouter/deepseek/deepseek-v4-pro": 5338, "openai/gpt-5.4": 8795 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T16-36-59-00-00_healthbench-professional-consult_D8Z93ZWFtkR4CDbDBLZvBT.eval", "task": "healthbench_professional_consult", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 236, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 94, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T16:36:59+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1888, "total_samples": 1888, "metrics": { "_scorer/bootstrap_score": 0.4864, "_scorer/std": 0.0381, "_scorer/criteria_met_rate": 0.6351, "_scorer/use_case_consult_score": 0.6058, "_scorer/use_case_consult_std": 0.0292, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/type_good_faith_score": 0.6676, "_scorer/type_good_faith_std": 0.0355, "_scorer/type_good_faith_n_samples": 143.0, "_scorer/type_red_teaming_score": 0.5109, "_scorer/type_red_teaming_std": 0.0504, "_scorer/type_red_teaming_n_samples": 93.0, "_scorer/difficulty_typical_score": 0.7806, "_scorer/difficulty_typical_std": 0.0353, "_scorer/difficulty_typical_n_samples": 84.0, "_scorer/difficulty_difficult_score": 0.5093, "_scorer/difficulty_difficult_std": 0.0374, "_scorer/difficulty_difficult_n_samples": 152.0, "_scorer/specialty_ent_score": 0.8271, "_scorer/specialty_ent_std": 0.0948, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.4375, "_scorer/specialty_gi_std": 0.1593, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.6133, "_scorer/specialty_id_std": 0.1398, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.7406, "_scorer/specialty_anesthesia_std": 0.1203, "_scorer/specialty_anesthesia_n_samples": 10.0, "_scorer/specialty_cards_score": 0.3748, "_scorer/specialty_cards_std": 0.0921, "_scorer/specialty_cards_n_samples": 23.0, "_scorer/specialty_derm_score": 0.6271, "_scorer/specialty_derm_std": 0.121, "_scorer/specialty_derm_n_samples": 14.0, "_scorer/specialty_emergency_score": 0.7046, "_scorer/specialty_emergency_std": 0.1229, "_scorer/specialty_emergency_n_samples": 8.0, "_scorer/specialty_endo_score": 0.7143, "_scorer/specialty_endo_std": 0.1406, "_scorer/specialty_endo_n_samples": 9.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.4533, "_scorer/specialty_heme/onc_std": 0.1513, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.3333, "_scorer/specialty_medicine_std": 0.1901, "_scorer/specialty_medicine_n_samples": 6.0, "_scorer/specialty_nephro_score": 0.8889, "_scorer/specialty_nephro_std": 0.1006, "_scorer/specialty_nephro_n_samples": 9.0, "_scorer/specialty_neuro_score": 0.7397, "_scorer/specialty_neuro_std": 0.0941, "_scorer/specialty_neuro_n_samples": 15.0, "_scorer/specialty_neurosurg_score": 0.6667, "_scorer/specialty_neurosurg_std": 0.1924, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.6429, "_scorer/specialty_obgyn_std": 0.1007, "_scorer/specialty_obgyn_n_samples": 18.0, "_scorer/specialty_ophtho_score": 0.52, "_scorer/specialty_ophtho_std": 0.1519, "_scorer/specialty_ophtho_n_samples": 9.0, "_scorer/specialty_ortho_score": 0.6239, "_scorer/specialty_ortho_std": 0.1343, "_scorer/specialty_ortho_n_samples": 13.0, "_scorer/specialty_other_score": 0.5, "_scorer/specialty_other_std": 0.2045, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_path_score": 0.3333, "_scorer/specialty_path_std": 0.2679, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.3277, "_scorer/specialty_peds_std": 0.109, "_scorer/specialty_peds_n_samples": 14.0, "_scorer/specialty_primary_score": 0.8676, "_scorer/specialty_primary_std": 0.1125, "_scorer/specialty_primary_n_samples": 4.0, "_scorer/specialty_psychiatry_score": 0.572, "_scorer/specialty_psychiatry_std": 0.1582, "_scorer/specialty_psychiatry_n_samples": 7.0, "_scorer/specialty_pulm_score": 0.75, "_scorer/specialty_pulm_std": 0.1573, "_scorer/specialty_pulm_n_samples": 6.0, "_scorer/specialty_rads_score": 0.6429, "_scorer/specialty_rads_std": 0.1715, "_scorer/specialty_rads_n_samples": 7.0, "_scorer/specialty_rheum_score": 0.6, "_scorer/specialty_rheum_std": 0.2161, "_scorer/specialty_rheum_n_samples": 5.0, "_scorer/specialty_surgery_score": 0.9, "_scorer/specialty_surgery_std": 0.0945, "_scorer/specialty_surgery_n_samples": 10.0, "_scorer/specialty_uro_score": 0.75, "_scorer/specialty_uro_std": 0.2109, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_good_faith_typical_score": 0.7806, "_scorer/source_slice_good_faith_typical_std": 0.0351, "_scorer/source_slice_good_faith_typical_n_samples": 84.0, "_scorer/source_slice_good_faith_difficult_score": 0.5066, "_scorer/source_slice_good_faith_difficult_std": 0.0618, "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, "_scorer/source_slice_red_teaming_difficult_score": 0.5109, "_scorer/source_slice_red_teaming_difficult_std": 0.0501, "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, "healthbench_professional_score/mean": 0.51, "healthbench_professional_score_length_adjusted/mean": 0.4864 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T16-37-37-00-00_healthbench-professional-writing_joZQNJ9ZAZidpUaSpEqyvS.eval", "task": "healthbench_professional_writing", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 142, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 56, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T16:37:37+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1136, "total_samples": 1136, "metrics": { "_scorer/bootstrap_score": 0.3596, "_scorer/std": 0.0455, "_scorer/criteria_met_rate": 0.5657, "_scorer/use_case_writing_score": 0.5245, "_scorer/use_case_writing_std": 0.0384, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_red_teaming_score": 0.3623, "_scorer/type_red_teaming_std": 0.0488, "_scorer/type_red_teaming_n_samples": 91.0, "_scorer/type_good_faith_score": 0.8139, "_scorer/type_good_faith_std": 0.0425, "_scorer/type_good_faith_n_samples": 51.0, "_scorer/difficulty_difficult_score": 0.3903, "_scorer/difficulty_difficult_std": 0.0486, "_scorer/difficulty_difficult_n_samples": 96.0, "_scorer/difficulty_typical_score": 0.8046, "_scorer/difficulty_typical_std": 0.0437, "_scorer/difficulty_typical_n_samples": 46.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.3567, "_scorer/specialty_ent_n_samples": 2.0, "_scorer/specialty_gi_score": 0.4, "_scorer/specialty_gi_std": 0.2175, "_scorer/specialty_gi_n_samples": 5.0, "_scorer/specialty_id_score": 0.275, "_scorer/specialty_id_std": 0.1196, "_scorer/specialty_id_n_samples": 8.0, "_scorer/specialty_anesthesia_score": 0.3061, "_scorer/specialty_anesthesia_std": 0.1608, "_scorer/specialty_anesthesia_n_samples": 4.0, "_scorer/specialty_cards_score": 0.6667, "_scorer/specialty_cards_std": 0.1905, "_scorer/specialty_cards_n_samples": 6.0, "_scorer/specialty_derm_score": 0.3151, "_scorer/specialty_derm_std": 0.1462, "_scorer/specialty_derm_n_samples": 7.0, "_scorer/specialty_emergency_score": 0.5089, "_scorer/specialty_emergency_std": 0.1786, "_scorer/specialty_emergency_n_samples": 7.0, "_scorer/specialty_endo_score": 0.5474, "_scorer/specialty_endo_std": 0.2038, "_scorer/specialty_endo_n_samples": 5.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.5169, "_scorer/specialty_heme/onc_std": 0.14, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.3333, "_scorer/specialty_medicine_std": 0.2712, "_scorer/specialty_medicine_n_samples": 3.0, "_scorer/specialty_nephro_score": 1.0, "_scorer/specialty_nephro_std": 0.0, "_scorer/specialty_nephro_n_samples": 6.0, "_scorer/specialty_neuro_score": 0.6111, "_scorer/specialty_neuro_std": 0.1281, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.25, "_scorer/specialty_neurosurg_std": 0.213, "_scorer/specialty_neurosurg_n_samples": 4.0, "_scorer/specialty_obgyn_score": 0.6397, "_scorer/specialty_obgyn_std": 0.1218, "_scorer/specialty_obgyn_n_samples": 12.0, "_scorer/specialty_ophtho_score": 0.475, "_scorer/specialty_ophtho_std": 0.1995, "_scorer/specialty_ophtho_n_samples": 5.0, "_scorer/specialty_ortho_score": 0.2778, "_scorer/specialty_ortho_std": 0.2099, "_scorer/specialty_ortho_n_samples": 4.0, "_scorer/specialty_other_score": 0.4167, "_scorer/specialty_other_std": 0.1612, "_scorer/specialty_other_n_samples": 8.0, "_scorer/specialty_peds_score": 0.731, "_scorer/specialty_peds_std": 0.132, "_scorer/specialty_peds_n_samples": 8.0, "_scorer/specialty_primary_score": 0.6667, "_scorer/specialty_primary_std": 0.2857, "_scorer/specialty_primary_n_samples": 3.0, "_scorer/specialty_psychiatry_score": 0.6278, "_scorer/specialty_psychiatry_std": 0.1312, "_scorer/specialty_psychiatry_n_samples": 12.0, "_scorer/specialty_pulm_score": 1.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 2.0, "_scorer/specialty_rads_score": 1.0, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 1.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 2.0, "_scorer/specialty_surgery_score": 0.25, "_scorer/specialty_surgery_std": 0.2109, "_scorer/specialty_surgery_n_samples": 4.0, "_scorer/specialty_uro_score": 0.5, "_scorer/specialty_uro_std": 0.2527, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_red_teaming_difficult_score": 0.3623, "_scorer/source_slice_red_teaming_difficult_std": 0.0488, "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, "_scorer/source_slice_good_faith_difficult_score": 0.9, "_scorer/source_slice_good_faith_difficult_std": 0.0875, "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, "_scorer/source_slice_good_faith_typical_score": 0.8046, "_scorer/source_slice_good_faith_typical_std": 0.0453, "_scorer/source_slice_good_faith_typical_n_samples": 46.0, "healthbench_professional_score/mean": 0.4062, "healthbench_professional_score_length_adjusted/mean": 0.3596 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T16-37-58-00-00_healthbench-professional-research_3Vd4jhz6oiMQTsC2Mhw849.eval", "task": "healthbench_professional_research", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 147, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 58, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T16:37:58+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1176, "total_samples": 1176, "metrics": { "_scorer/bootstrap_score": 0.5792, "_scorer/std": 0.0326, "_scorer/criteria_met_rate": 0.6963, "_scorer/use_case_research_score": 0.7055, "_scorer/use_case_research_std": 0.0298, "_scorer/use_case_research_n_samples": 147.0, "_scorer/type_red_teaming_score": 1.0, "_scorer/type_red_teaming_std": 0.0, "_scorer/type_red_teaming_n_samples": 7.0, "_scorer/type_good_faith_score": 0.6908, "_scorer/type_good_faith_std": 0.0319, "_scorer/type_good_faith_n_samples": 140.0, "_scorer/difficulty_difficult_score": 0.5952, "_scorer/difficulty_difficult_std": 0.1043, "_scorer/difficulty_difficult_n_samples": 21.0, "_scorer/difficulty_typical_score": 0.7239, "_scorer/difficulty_typical_std": 0.0317, "_scorer/difficulty_typical_n_samples": 126.0, "_scorer/specialty_ent_score": 1.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 1.0, "_scorer/specialty_gi_score": 0.4165, "_scorer/specialty_gi_std": 0.1281, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.7302, "_scorer/specialty_id_std": 0.0844, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_anesthesia_score": 0.8214, "_scorer/specialty_anesthesia_std": 0.1203, "_scorer/specialty_anesthesia_n_samples": 8.0, "_scorer/specialty_cards_score": 0.748, "_scorer/specialty_cards_std": 0.0905, "_scorer/specialty_cards_n_samples": 14.0, "_scorer/specialty_derm_score": 1.0, "_scorer/specialty_derm_std": 0.0, "_scorer/specialty_derm_n_samples": 4.0, "_scorer/specialty_emergency_score": 1.0, "_scorer/specialty_emergency_std": 0.0, "_scorer/specialty_emergency_n_samples": 1.0, "_scorer/specialty_endo_score": 1.0, "_scorer/specialty_endo_std": 0.0, "_scorer/specialty_endo_n_samples": 6.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 2.0, "_scorer/specialty_heme/onc_score": 0.5922, "_scorer/specialty_heme/onc_std": 0.1207, "_scorer/specialty_heme/onc_n_samples": 7.0, "_scorer/specialty_medicine_score": 0.5476, "_scorer/specialty_medicine_std": 0.165, "_scorer/specialty_medicine_n_samples": 8.0, "_scorer/specialty_nephro_score": 0.6752, "_scorer/specialty_nephro_std": 0.1315, "_scorer/specialty_nephro_n_samples": 7.0, "_scorer/specialty_neuro_score": 0.7423, "_scorer/specialty_neuro_std": 0.075, "_scorer/specialty_neuro_n_samples": 19.0, "_scorer/specialty_neurosurg_score": 0.0667, "_scorer/specialty_neurosurg_std": 0.0, "_scorer/specialty_neurosurg_n_samples": 1.0, "_scorer/specialty_obgyn_score": 0.697, "_scorer/specialty_obgyn_std": 0.0934, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 1.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 4.0, "_scorer/specialty_ortho_score": 0.2036, "_scorer/specialty_ortho_std": 0.0827, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.5625, "_scorer/specialty_other_std": 0.1729, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_peds_score": 0.7619, "_scorer/specialty_peds_std": 0.1562, "_scorer/specialty_peds_n_samples": 6.0, "_scorer/specialty_psychiatry_score": 0.8125, "_scorer/specialty_psychiatry_std": 0.1149, "_scorer/specialty_psychiatry_n_samples": 10.0, "_scorer/specialty_pulm_score": 0.6198, "_scorer/specialty_pulm_std": 0.1396, "_scorer/specialty_pulm_n_samples": 4.0, "_scorer/specialty_rheum_score": 1.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 1.0, "_scorer/specialty_surgery_score": 0.8333, "_scorer/specialty_surgery_std": 0.1358, "_scorer/specialty_surgery_n_samples": 3.0, "_scorer/specialty_uro_score": 0.7, "_scorer/specialty_uro_std": 0.1781, "_scorer/specialty_uro_n_samples": 5.0, "_scorer/source_slice_red_teaming_difficult_score": 1.0, "_scorer/source_slice_red_teaming_difficult_std": 0.0, "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, "_scorer/source_slice_good_faith_typical_score": 0.7239, "_scorer/source_slice_good_faith_typical_std": 0.0311, "_scorer/source_slice_good_faith_typical_n_samples": 126.0, "_scorer/source_slice_good_faith_difficult_score": 0.3929, "_scorer/source_slice_good_faith_difficult_std": 0.1233, "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, "healthbench_professional_score/mean": 0.6799, "healthbench_professional_score_length_adjusted/mean": 0.5792 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T16-38-22-00-00_healthbench-professional-red-teaming_Xund3mwQvSqTJfxno7icGe.eval", "task": "healthbench_professional_red_teaming", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 191, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 76, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T16:38:22+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1528, "total_samples": 1528, "metrics": { "_scorer/bootstrap_score": 0.2819, "_scorer/std": 0.0444, "_scorer/criteria_met_rate": 0.4959, "_scorer/use_case_consult_score": 0.5109, "_scorer/use_case_consult_std": 0.0509, "_scorer/use_case_consult_n_samples": 93.0, "_scorer/use_case_research_score": 1.0, "_scorer/use_case_research_std": 0.0, "_scorer/use_case_research_n_samples": 7.0, "_scorer/use_case_writing_score": 0.3623, "_scorer/use_case_writing_std": 0.0488, "_scorer/use_case_writing_n_samples": 91.0, "_scorer/type_red_teaming_score": 0.458, "_scorer/type_red_teaming_std": 0.034, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_difficult_score": 0.458, "_scorer/difficulty_difficult_std": 0.0349, "_scorer/difficulty_difficult_n_samples": 191.0, "_scorer/specialty_ent_score": 0.75, "_scorer/specialty_ent_std": 0.217, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.25, "_scorer/specialty_gi_std": 0.1523, "_scorer/specialty_gi_n_samples": 8.0, "_scorer/specialty_id_score": 0.32, "_scorer/specialty_id_std": 0.1214, "_scorer/specialty_id_n_samples": 10.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 1.0, "_scorer/specialty_anesthesia_score": 0.5238, "_scorer/specialty_anesthesia_std": 0.1767, "_scorer/specialty_anesthesia_n_samples": 7.0, "_scorer/specialty_cards_score": 0.303, "_scorer/specialty_cards_std": 0.1237, "_scorer/specialty_cards_n_samples": 11.0, "_scorer/specialty_derm_score": 0.3714, "_scorer/specialty_derm_std": 0.1343, "_scorer/specialty_derm_n_samples": 11.0, "_scorer/specialty_emergency_score": 0.5966, "_scorer/specialty_emergency_std": 0.1383, "_scorer/specialty_emergency_n_samples": 11.0, "_scorer/specialty_endo_score": 0.5429, "_scorer/specialty_endo_std": 0.1471, "_scorer/specialty_endo_n_samples": 10.0, "_scorer/specialty_genetics_score": 0.6667, "_scorer/specialty_genetics_std": 0.2765, "_scorer/specialty_genetics_n_samples": 3.0, "_scorer/specialty_heme/onc_score": 0.4067, "_scorer/specialty_heme/onc_std": 0.138, "_scorer/specialty_heme/onc_n_samples": 11.0, "_scorer/specialty_medicine_score": 0.2857, "_scorer/specialty_medicine_std": 0.1691, "_scorer/specialty_medicine_n_samples": 7.0, "_scorer/specialty_nephro_score": 0.75, "_scorer/specialty_nephro_std": 0.2217, "_scorer/specialty_nephro_n_samples": 4.0, "_scorer/specialty_neuro_score": 0.6667, "_scorer/specialty_neuro_std": 0.1524, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.3333, "_scorer/specialty_neurosurg_std": 0.1956, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.5215, "_scorer/specialty_obgyn_std": 0.1434, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.3333, "_scorer/specialty_ophtho_std": 0.2704, "_scorer/specialty_ophtho_n_samples": 3.0, "_scorer/specialty_ortho_score": 0.3016, "_scorer/specialty_ortho_std": 0.1682, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.2222, "_scorer/specialty_other_std": 0.1392, "_scorer/specialty_other_n_samples": 9.0, "_scorer/specialty_path_score": 1.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 1.0, "_scorer/specialty_peds_score": 0.3407, "_scorer/specialty_peds_std": 0.1202, "_scorer/specialty_peds_n_samples": 10.0, "_scorer/specialty_primary_score": 0.8, "_scorer/specialty_primary_std": 0.1815, "_scorer/specialty_primary_n_samples": 5.0, "_scorer/specialty_psychiatry_score": 0.3333, "_scorer/specialty_psychiatry_std": 0.1553, "_scorer/specialty_psychiatry_n_samples": 9.0, "_scorer/specialty_pulm_score": 0.8, "_scorer/specialty_pulm_std": 0.1838, "_scorer/specialty_pulm_n_samples": 5.0, "_scorer/specialty_rads_score": 0.8, "_scorer/specialty_rads_std": 0.1706, "_scorer/specialty_rads_n_samples": 5.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 4.0, "_scorer/specialty_surgery_score": 0.5714, "_scorer/specialty_surgery_std": 0.1882, "_scorer/specialty_surgery_n_samples": 7.0, "_scorer/specialty_uro_score": 0.6667, "_scorer/specialty_uro_std": 0.2724, "_scorer/specialty_uro_n_samples": 3.0, "_scorer/source_slice_red_teaming_difficult_score": 0.458, "_scorer/source_slice_red_teaming_difficult_std": 0.0353, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.2992, "healthbench_professional_score_length_adjusted/mean": 0.2819 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T16-38-52-00-00_healthbench-professional-physician-baseline_XADueRF5xwkSmDPWc4LtNf.eval", "task": "healthbench_professional_physician_baseline", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T16:38:52+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4387, "_scorer/std": 0.0244, "_scorer/criteria_met_rate": 0.5044, "_scorer/use_case_consult_score": 0.4843, "_scorer/use_case_consult_std": 0.0284, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.59, "_scorer/use_case_research_std": 0.0312, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.449, "_scorer/use_case_writing_std": 0.0356, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5561, "_scorer/type_good_faith_std": 0.0211, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4137, "_scorer/type_red_teaming_std": 0.0327, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5803, "_scorer/difficulty_typical_std": 0.023, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.432, "_scorer/difficulty_difficult_std": 0.0285, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.175, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.5427, "_scorer/specialty_gi_std": 0.0933, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.5113, "_scorer/specialty_id_std": 0.0732, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6229, "_scorer/specialty_anesthesia_std": 0.087, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4092, "_scorer/specialty_cards_std": 0.0625, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6483, "_scorer/specialty_derm_std": 0.0798, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4549, "_scorer/specialty_emergency_std": 0.1188, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5268, "_scorer/specialty_endo_std": 0.0835, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.7955, "_scorer/specialty_genetics_std": 0.105, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.5019, "_scorer/specialty_heme/onc_std": 0.0808, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.6713, "_scorer/specialty_medicine_std": 0.098, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4311, "_scorer/specialty_nephro_std": 0.0796, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5592, "_scorer/specialty_neuro_std": 0.0605, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.3242, "_scorer/specialty_neurosurg_std": 0.1175, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4791, "_scorer/specialty_obgyn_std": 0.0641, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3167, "_scorer/specialty_ophtho_std": 0.0976, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.3507, "_scorer/specialty_ortho_std": 0.0825, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3541, "_scorer/specialty_other_std": 0.0835, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.6667, "_scorer/specialty_path_std": 0.2716, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.5363, "_scorer/specialty_peds_std": 0.0853, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1627, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5115, "_scorer/specialty_psychiatry_std": 0.0697, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5848, "_scorer/specialty_pulm_std": 0.1138, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.3787, "_scorer/specialty_rads_std": 0.1538, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.6964, "_scorer/specialty_rheum_std": 0.1524, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.498, "_scorer/specialty_surgery_std": 0.1142, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6175, "_scorer/specialty_uro_std": 0.1068, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5803, "_scorer/source_slice_good_faith_typical_std": 0.0233, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.4768, "_scorer/source_slice_good_faith_difficult_std": 0.0526, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4137, "_scorer/source_slice_red_teaming_difficult_std": 0.0324, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.4429, "healthbench_professional_score_length_adjusted/mean": 0.4387 }, "model_usage_models": [ "openai/gpt-5.4" ], "model_usage_total": { "openai/gpt-5.4": 17204312 }, "error": null, "solver": "physician_baseline_solver" }, { "space": "hs-prof-subsets", "file": "2026-07-24T17-19-14-00-00_healthbench-professional-consult_hZgDXu6oT2MuZrn4L7msXN.eval", "task": "healthbench_professional_consult", "task_version": 0, "task_file": "healthbench_professional.py", "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 236, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 94, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T17:19:14+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1888, "total_samples": 1888, "metrics": { "_scorer/bootstrap_score": 0.4704, "_scorer/std": 0.035, "_scorer/criteria_met_rate": 0.6144, "_scorer/use_case_consult_score": 0.6051, "_scorer/use_case_consult_std": 0.0281, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/type_good_faith_score": 0.6569, "_scorer/type_good_faith_std": 0.0351, "_scorer/type_good_faith_n_samples": 143.0, "_scorer/type_red_teaming_score": 0.5254, "_scorer/type_red_teaming_std": 0.0477, "_scorer/type_red_teaming_n_samples": 93.0, "_scorer/difficulty_typical_score": 0.7495, "_scorer/difficulty_typical_std": 0.0356, "_scorer/difficulty_typical_n_samples": 84.0, "_scorer/difficulty_difficult_score": 0.5253, "_scorer/difficulty_difficult_std": 0.0375, "_scorer/difficulty_difficult_n_samples": 152.0, "_scorer/specialty_ent_score": 0.6937, "_scorer/specialty_ent_std": 0.2083, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.4459, "_scorer/specialty_gi_std": 0.1624, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.75, "_scorer/specialty_id_std": 0.1559, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_allergy_score": 0.5, "_scorer/specialty_allergy_std": 0.3575, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.8275, "_scorer/specialty_anesthesia_std": 0.0941, "_scorer/specialty_anesthesia_n_samples": 10.0, "_scorer/specialty_cards_score": 0.4499, "_scorer/specialty_cards_std": 0.088, "_scorer/specialty_cards_n_samples": 23.0, "_scorer/specialty_derm_score": 0.7283, "_scorer/specialty_derm_std": 0.111, "_scorer/specialty_derm_n_samples": 14.0, "_scorer/specialty_emergency_score": 0.6838, "_scorer/specialty_emergency_std": 0.1457, "_scorer/specialty_emergency_n_samples": 8.0, "_scorer/specialty_endo_score": 0.7143, "_scorer/specialty_endo_std": 0.1396, "_scorer/specialty_endo_n_samples": 9.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.406, "_scorer/specialty_heme/onc_std": 0.1395, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.4993, "_scorer/specialty_medicine_std": 0.1594, "_scorer/specialty_medicine_n_samples": 6.0, "_scorer/specialty_nephro_score": 0.6942, "_scorer/specialty_nephro_std": 0.1144, "_scorer/specialty_nephro_n_samples": 9.0, "_scorer/specialty_neuro_score": 0.7375, "_scorer/specialty_neuro_std": 0.0976, "_scorer/specialty_neuro_n_samples": 15.0, "_scorer/specialty_neurosurg_score": 0.7588, "_scorer/specialty_neurosurg_std": 0.1038, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.5495, "_scorer/specialty_obgyn_std": 0.0928, "_scorer/specialty_obgyn_n_samples": 18.0, "_scorer/specialty_ophtho_score": 0.4089, "_scorer/specialty_ophtho_std": 0.15, "_scorer/specialty_ophtho_n_samples": 9.0, "_scorer/specialty_ortho_score": 0.8103, "_scorer/specialty_ortho_std": 0.1004, "_scorer/specialty_ortho_n_samples": 13.0, "_scorer/specialty_other_score": 0.6667, "_scorer/specialty_other_std": 0.1911, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_path_score": 0.1905, "_scorer/specialty_path_std": 0.1574, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.2979, "_scorer/specialty_peds_std": 0.1081, "_scorer/specialty_peds_n_samples": 14.0, "_scorer/specialty_primary_score": 0.6176, "_scorer/specialty_primary_std": 0.2146, "_scorer/specialty_primary_n_samples": 4.0, "_scorer/specialty_psychiatry_score": 0.6392, "_scorer/specialty_psychiatry_std": 0.1309, "_scorer/specialty_psychiatry_n_samples": 7.0, "_scorer/specialty_pulm_score": 0.5, "_scorer/specialty_pulm_std": 0.1634, "_scorer/specialty_pulm_n_samples": 6.0, "_scorer/specialty_rads_score": 0.7101, "_scorer/specialty_rads_std": 0.1344, "_scorer/specialty_rads_n_samples": 7.0, "_scorer/specialty_rheum_score": 0.6, "_scorer/specialty_rheum_std": 0.2164, "_scorer/specialty_rheum_n_samples": 5.0, "_scorer/specialty_surgery_score": 0.8, "_scorer/specialty_surgery_std": 0.1269, "_scorer/specialty_surgery_n_samples": 10.0, "_scorer/specialty_uro_score": 0.75, "_scorer/specialty_uro_std": 0.2207, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_good_faith_typical_score": 0.7495, "_scorer/source_slice_good_faith_typical_std": 0.0359, "_scorer/source_slice_good_faith_typical_n_samples": 84.0, "_scorer/source_slice_good_faith_difficult_score": 0.525, "_scorer/source_slice_good_faith_difficult_std": 0.0609, "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, "_scorer/source_slice_red_teaming_difficult_score": 0.5254, "_scorer/source_slice_red_teaming_difficult_std": 0.0474, "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, "healthbench_professional_score/mean": 0.4906, "healthbench_professional_score_length_adjusted/mean": 0.4704 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T17-19-42-00-00_healthbench-professional-writing_haJJesaYyVNeNRLHnAi8HD.eval", "task": "healthbench_professional_writing", "task_version": 0, "task_file": "healthbench_professional.py", "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 142, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 56, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T17:19:42+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1136, "total_samples": 1136, "metrics": { "_scorer/bootstrap_score": 0.3615, "_scorer/std": 0.0505, "_scorer/criteria_met_rate": 0.5814, "_scorer/use_case_writing_score": 0.5091, "_scorer/use_case_writing_std": 0.0382, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_red_teaming_score": 0.348, "_scorer/type_red_teaming_std": 0.0488, "_scorer/type_red_teaming_n_samples": 91.0, "_scorer/type_good_faith_score": 0.7967, "_scorer/type_good_faith_std": 0.0431, "_scorer/type_good_faith_n_samples": 51.0, "_scorer/difficulty_difficult_score": 0.3767, "_scorer/difficulty_difficult_std": 0.0452, "_scorer/difficulty_difficult_n_samples": 96.0, "_scorer/difficulty_typical_score": 0.7854, "_scorer/difficulty_typical_std": 0.0478, "_scorer/difficulty_typical_n_samples": 46.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.3499, "_scorer/specialty_ent_n_samples": 2.0, "_scorer/specialty_gi_score": 0.5059, "_scorer/specialty_gi_std": 0.1975, "_scorer/specialty_gi_n_samples": 5.0, "_scorer/specialty_id_score": 0.4, "_scorer/specialty_id_std": 0.1399, "_scorer/specialty_id_n_samples": 8.0, "_scorer/specialty_anesthesia_score": 0.5, "_scorer/specialty_anesthesia_std": 0.2552, "_scorer/specialty_anesthesia_n_samples": 4.0, "_scorer/specialty_cards_score": 0.625, "_scorer/specialty_cards_std": 0.1838, "_scorer/specialty_cards_n_samples": 6.0, "_scorer/specialty_derm_score": 0.3151, "_scorer/specialty_derm_std": 0.1496, "_scorer/specialty_derm_n_samples": 7.0, "_scorer/specialty_emergency_score": 0.2857, "_scorer/specialty_emergency_std": 0.1775, "_scorer/specialty_emergency_n_samples": 7.0, "_scorer/specialty_endo_score": 0.3474, "_scorer/specialty_endo_std": 0.1936, "_scorer/specialty_endo_n_samples": 5.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.5111, "_scorer/specialty_heme/onc_std": 0.1422, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.6667, "_scorer/specialty_medicine_std": 0.272, "_scorer/specialty_medicine_n_samples": 3.0, "_scorer/specialty_nephro_score": 0.9053, "_scorer/specialty_nephro_std": 0.054, "_scorer/specialty_nephro_n_samples": 6.0, "_scorer/specialty_neuro_score": 0.6685, "_scorer/specialty_neuro_std": 0.1344, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.25, "_scorer/specialty_neurosurg_std": 0.2219, "_scorer/specialty_neurosurg_n_samples": 4.0, "_scorer/specialty_obgyn_score": 0.5687, "_scorer/specialty_obgyn_std": 0.1221, "_scorer/specialty_obgyn_n_samples": 12.0, "_scorer/specialty_ophtho_score": 0.475, "_scorer/specialty_ophtho_std": 0.2006, "_scorer/specialty_ophtho_n_samples": 5.0, "_scorer/specialty_ortho_score": 0.2778, "_scorer/specialty_ortho_std": 0.2095, "_scorer/specialty_ortho_n_samples": 4.0, "_scorer/specialty_other_score": 0.4792, "_scorer/specialty_other_std": 0.152, "_scorer/specialty_other_n_samples": 8.0, "_scorer/specialty_peds_score": 0.625, "_scorer/specialty_peds_std": 0.1493, "_scorer/specialty_peds_n_samples": 8.0, "_scorer/specialty_primary_score": 0.6667, "_scorer/specialty_primary_std": 0.2683, "_scorer/specialty_primary_n_samples": 3.0, "_scorer/specialty_psychiatry_score": 0.6808, "_scorer/specialty_psychiatry_std": 0.1225, "_scorer/specialty_psychiatry_n_samples": 12.0, "_scorer/specialty_pulm_score": 1.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 2.0, "_scorer/specialty_rads_score": 1.0, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 1.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 2.0, "_scorer/specialty_surgery_score": 0.25, "_scorer/specialty_surgery_std": 0.2153, "_scorer/specialty_surgery_n_samples": 4.0, "_scorer/specialty_uro_score": 0.25, "_scorer/specialty_uro_std": 0.2181, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_red_teaming_difficult_score": 0.348, "_scorer/source_slice_red_teaming_difficult_std": 0.0463, "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, "_scorer/source_slice_good_faith_difficult_score": 0.9, "_scorer/source_slice_good_faith_difficult_std": 0.0893, "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, "_scorer/source_slice_good_faith_typical_score": 0.7854, "_scorer/source_slice_good_faith_typical_std": 0.0452, "_scorer/source_slice_good_faith_typical_n_samples": 46.0, "healthbench_professional_score/mean": 0.3947, "healthbench_professional_score_length_adjusted/mean": 0.3615 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T17-19-59-00-00_healthbench-professional-research_7RUfdNFLnH7EpAC92NeL7e.eval", "task": "healthbench_professional_research", "task_version": 0, "task_file": "healthbench_professional.py", "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 147, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 58, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T17:19:59+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1176, "total_samples": 1176, "metrics": { "_scorer/bootstrap_score": 0.6107, "_scorer/std": 0.0317, "_scorer/criteria_met_rate": 0.6757, "_scorer/use_case_research_score": 0.6739, "_scorer/use_case_research_std": 0.0313, "_scorer/use_case_research_n_samples": 147.0, "_scorer/type_red_teaming_score": 1.0, "_scorer/type_red_teaming_std": 0.0, "_scorer/type_red_teaming_n_samples": 7.0, "_scorer/type_good_faith_score": 0.6576, "_scorer/type_good_faith_std": 0.0331, "_scorer/type_good_faith_n_samples": 140.0, "_scorer/difficulty_difficult_score": 0.5476, "_scorer/difficulty_difficult_std": 0.1061, "_scorer/difficulty_difficult_n_samples": 21.0, "_scorer/difficulty_typical_score": 0.6949, "_scorer/difficulty_typical_std": 0.0316, "_scorer/difficulty_typical_n_samples": 126.0, "_scorer/specialty_ent_score": 1.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 1.0, "_scorer/specialty_gi_score": 0.3108, "_scorer/specialty_gi_std": 0.1035, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.7857, "_scorer/specialty_id_std": 0.0927, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_anesthesia_score": 0.8214, "_scorer/specialty_anesthesia_std": 0.0802, "_scorer/specialty_anesthesia_n_samples": 8.0, "_scorer/specialty_cards_score": 0.7769, "_scorer/specialty_cards_std": 0.0774, "_scorer/specialty_cards_n_samples": 14.0, "_scorer/specialty_derm_score": 0.75, "_scorer/specialty_derm_std": 0.224, "_scorer/specialty_derm_n_samples": 4.0, "_scorer/specialty_emergency_score": 1.0, "_scorer/specialty_emergency_std": 0.0, "_scorer/specialty_emergency_n_samples": 1.0, "_scorer/specialty_endo_score": 0.94, "_scorer/specialty_endo_std": 0.0544, "_scorer/specialty_endo_n_samples": 6.0, "_scorer/specialty_genetics_score": 0.8409, "_scorer/specialty_genetics_std": 0.1137, "_scorer/specialty_genetics_n_samples": 2.0, "_scorer/specialty_heme/onc_score": 0.5498, "_scorer/specialty_heme/onc_std": 0.1444, "_scorer/specialty_heme/onc_n_samples": 7.0, "_scorer/specialty_medicine_score": 0.5817, "_scorer/specialty_medicine_std": 0.1606, "_scorer/specialty_medicine_n_samples": 8.0, "_scorer/specialty_nephro_score": 0.8031, "_scorer/specialty_nephro_std": 0.1318, "_scorer/specialty_nephro_n_samples": 7.0, "_scorer/specialty_neuro_score": 0.7282, "_scorer/specialty_neuro_std": 0.0731, "_scorer/specialty_neuro_n_samples": 19.0, "_scorer/specialty_neurosurg_score": 0.0667, "_scorer/specialty_neurosurg_std": 0.0, "_scorer/specialty_neurosurg_n_samples": 1.0, "_scorer/specialty_obgyn_score": 0.5, "_scorer/specialty_obgyn_std": 0.1406, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 1.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 4.0, "_scorer/specialty_ortho_score": 0.3702, "_scorer/specialty_ortho_std": 0.1636, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.6267, "_scorer/specialty_other_std": 0.1436, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_peds_score": 0.6613, "_scorer/specialty_peds_std": 0.1598, "_scorer/specialty_peds_n_samples": 6.0, "_scorer/specialty_psychiatry_score": 0.675, "_scorer/specialty_psychiatry_std": 0.1402, "_scorer/specialty_psychiatry_n_samples": 10.0, "_scorer/specialty_pulm_score": 0.5401, "_scorer/specialty_pulm_std": 0.1804, "_scorer/specialty_pulm_n_samples": 4.0, "_scorer/specialty_rheum_score": 1.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 1.0, "_scorer/specialty_surgery_score": 0.8333, "_scorer/specialty_surgery_std": 0.1347, "_scorer/specialty_surgery_n_samples": 3.0, "_scorer/specialty_uro_score": 0.5286, "_scorer/specialty_uro_std": 0.181, "_scorer/specialty_uro_n_samples": 5.0, "_scorer/source_slice_red_teaming_difficult_score": 1.0, "_scorer/source_slice_red_teaming_difficult_std": 0.0, "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, "_scorer/source_slice_good_faith_typical_score": 0.6949, "_scorer/source_slice_good_faith_typical_std": 0.0316, "_scorer/source_slice_good_faith_typical_n_samples": 126.0, "_scorer/source_slice_good_faith_difficult_score": 0.3214, "_scorer/source_slice_good_faith_difficult_std": 0.1197, "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, "healthbench_professional_score/mean": 0.6464, "healthbench_professional_score_length_adjusted/mean": 0.6107 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T17-20-17-00-00_healthbench-professional-red-teaming_DLPPa6kbs9VeKS6cfPDhdv.eval", "task": "healthbench_professional_red_teaming", "task_version": 0, "task_file": "healthbench_professional.py", "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 191, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 76, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T17:20:17+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1528, "total_samples": 1528, "metrics": { "_scorer/bootstrap_score": 0.2674, "_scorer/std": 0.0457, "_scorer/criteria_met_rate": 0.5191, "_scorer/use_case_consult_score": 0.5254, "_scorer/use_case_consult_std": 0.0469, "_scorer/use_case_consult_n_samples": 93.0, "_scorer/use_case_research_score": 1.0, "_scorer/use_case_research_std": 0.0, "_scorer/use_case_research_n_samples": 7.0, "_scorer/use_case_writing_score": 0.348, "_scorer/use_case_writing_std": 0.047, "_scorer/use_case_writing_n_samples": 91.0, "_scorer/type_red_teaming_score": 0.4583, "_scorer/type_red_teaming_std": 0.0328, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_difficult_score": 0.4583, "_scorer/difficulty_difficult_std": 0.0342, "_scorer/difficulty_difficult_n_samples": 191.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.2537, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.3235, "_scorer/specialty_gi_std": 0.1471, "_scorer/specialty_gi_n_samples": 8.0, "_scorer/specialty_id_score": 0.42, "_scorer/specialty_id_std": 0.1362, "_scorer/specialty_id_n_samples": 10.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 1.0, "_scorer/specialty_anesthesia_score": 0.5238, "_scorer/specialty_anesthesia_std": 0.172, "_scorer/specialty_anesthesia_n_samples": 7.0, "_scorer/specialty_cards_score": 0.4046, "_scorer/specialty_cards_std": 0.1274, "_scorer/specialty_cards_n_samples": 11.0, "_scorer/specialty_derm_score": 0.3714, "_scorer/specialty_derm_std": 0.1265, "_scorer/specialty_derm_n_samples": 11.0, "_scorer/specialty_emergency_score": 0.4545, "_scorer/specialty_emergency_std": 0.1481, "_scorer/specialty_emergency_n_samples": 11.0, "_scorer/specialty_endo_score": 0.4429, "_scorer/specialty_endo_std": 0.1471, "_scorer/specialty_endo_n_samples": 10.0, "_scorer/specialty_genetics_score": 0.3333, "_scorer/specialty_genetics_std": 0.2626, "_scorer/specialty_genetics_n_samples": 3.0, "_scorer/specialty_heme/onc_score": 0.361, "_scorer/specialty_heme/onc_std": 0.1306, "_scorer/specialty_heme/onc_n_samples": 11.0, "_scorer/specialty_medicine_score": 0.6471, "_scorer/specialty_medicine_std": 0.1607, "_scorer/specialty_medicine_n_samples": 7.0, "_scorer/specialty_nephro_score": 0.75, "_scorer/specialty_nephro_std": 0.2211, "_scorer/specialty_nephro_n_samples": 4.0, "_scorer/specialty_neuro_score": 0.6711, "_scorer/specialty_neuro_std": 0.1604, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.4255, "_scorer/specialty_neurosurg_std": 0.1475, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.4825, "_scorer/specialty_obgyn_std": 0.1368, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.3333, "_scorer/specialty_ophtho_std": 0.2679, "_scorer/specialty_ophtho_n_samples": 3.0, "_scorer/specialty_ortho_score": 0.5873, "_scorer/specialty_ortho_std": 0.178, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.3889, "_scorer/specialty_other_std": 0.1543, "_scorer/specialty_other_n_samples": 9.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 1.0, "_scorer/specialty_peds_score": 0.2641, "_scorer/specialty_peds_std": 0.1266, "_scorer/specialty_peds_n_samples": 10.0, "_scorer/specialty_primary_score": 0.6, "_scorer/specialty_primary_std": 0.2216, "_scorer/specialty_primary_n_samples": 5.0, "_scorer/specialty_psychiatry_score": 0.4967, "_scorer/specialty_psychiatry_std": 0.1586, "_scorer/specialty_psychiatry_n_samples": 9.0, "_scorer/specialty_pulm_score": 0.6, "_scorer/specialty_pulm_std": 0.2172, "_scorer/specialty_pulm_n_samples": 5.0, "_scorer/specialty_rads_score": 0.8941, "_scorer/specialty_rads_std": 0.0905, "_scorer/specialty_rads_n_samples": 5.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 4.0, "_scorer/specialty_surgery_score": 0.4286, "_scorer/specialty_surgery_std": 0.1844, "_scorer/specialty_surgery_n_samples": 7.0, "_scorer/specialty_uro_score": 0.3333, "_scorer/specialty_uro_std": 0.2713, "_scorer/specialty_uro_n_samples": 3.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4583, "_scorer/source_slice_red_teaming_difficult_std": 0.0337, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.2828, "healthbench_professional_score_length_adjusted/mean": 0.2674 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T17-20-42-00-00_healthbench-professional-physician-baseline_98adhjPPBGUtnnuJVmWXjn.eval", "task": "healthbench_professional_physician_baseline", "task_version": 0, "task_file": "healthbench_professional.py", "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T17:20:42+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4387, "_scorer/std": 0.023, "_scorer/criteria_met_rate": 0.5044, "_scorer/use_case_consult_score": 0.4843, "_scorer/use_case_consult_std": 0.0275, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.59, "_scorer/use_case_research_std": 0.0309, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.449, "_scorer/use_case_writing_std": 0.0372, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5561, "_scorer/type_good_faith_std": 0.0209, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4137, "_scorer/type_red_teaming_std": 0.0318, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5803, "_scorer/difficulty_typical_std": 0.0222, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.432, "_scorer/difficulty_difficult_std": 0.0271, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.176, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.5427, "_scorer/specialty_gi_std": 0.0962, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.5113, "_scorer/specialty_id_std": 0.0721, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6229, "_scorer/specialty_anesthesia_std": 0.0904, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4092, "_scorer/specialty_cards_std": 0.0608, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6483, "_scorer/specialty_derm_std": 0.0818, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4549, "_scorer/specialty_emergency_std": 0.1171, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5268, "_scorer/specialty_endo_std": 0.0814, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.7955, "_scorer/specialty_genetics_std": 0.1101, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.5019, "_scorer/specialty_heme/onc_std": 0.0834, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.6713, "_scorer/specialty_medicine_std": 0.1017, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4311, "_scorer/specialty_nephro_std": 0.0829, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5592, "_scorer/specialty_neuro_std": 0.0608, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.3242, "_scorer/specialty_neurosurg_std": 0.1113, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4791, "_scorer/specialty_obgyn_std": 0.0631, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3167, "_scorer/specialty_ophtho_std": 0.0948, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.3507, "_scorer/specialty_ortho_std": 0.0905, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3541, "_scorer/specialty_other_std": 0.0877, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.6667, "_scorer/specialty_path_std": 0.2746, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.5363, "_scorer/specialty_peds_std": 0.0828, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1582, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5115, "_scorer/specialty_psychiatry_std": 0.0696, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5848, "_scorer/specialty_pulm_std": 0.1163, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.3787, "_scorer/specialty_rads_std": 0.1382, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.6964, "_scorer/specialty_rheum_std": 0.1529, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.498, "_scorer/specialty_surgery_std": 0.1205, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6175, "_scorer/specialty_uro_std": 0.1159, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5803, "_scorer/source_slice_good_faith_typical_std": 0.0224, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.4768, "_scorer/source_slice_good_faith_difficult_std": 0.0513, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4137, "_scorer/source_slice_red_teaming_difficult_std": 0.0322, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.4429, "healthbench_professional_score_length_adjusted/mean": 0.4387 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "physician_baseline_solver" }, { "space": "hs-prof-subsets", "file": "2026-07-24T19-33-04-00-00_healthbench-professional-consult_dgWnf9y2uVjcWoDcyTm3Mm.eval", "task": "healthbench_professional_consult", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 236, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 94, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T19:33:04+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1888, "total_samples": 1888, "metrics": { "_scorer/bootstrap_score": 0.1536, "_scorer/std": 0.0318, "_scorer/criteria_met_rate": 0.4021, "_scorer/use_case_consult_score": 0.3313, "_scorer/use_case_consult_std": 0.027, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/type_good_faith_score": 0.3876, "_scorer/type_good_faith_std": 0.0362, "_scorer/type_good_faith_n_samples": 143.0, "_scorer/type_red_teaming_score": 0.2446, "_scorer/type_red_teaming_std": 0.0421, "_scorer/type_red_teaming_n_samples": 93.0, "_scorer/difficulty_typical_score": 0.4979, "_scorer/difficulty_typical_std": 0.0436, "_scorer/difficulty_typical_n_samples": 84.0, "_scorer/difficulty_difficult_score": 0.2391, "_scorer/difficulty_difficult_std": 0.0315, "_scorer/difficulty_difficult_n_samples": 152.0, "_scorer/specialty_ent_score": 0.6, "_scorer/specialty_ent_std": 0.2166, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.0833, "_scorer/specialty_id_std": 0.0759, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.3553, "_scorer/specialty_anesthesia_std": 0.1369, "_scorer/specialty_anesthesia_n_samples": 10.0, "_scorer/specialty_cards_score": 0.329, "_scorer/specialty_cards_std": 0.0898, "_scorer/specialty_cards_n_samples": 23.0, "_scorer/specialty_derm_score": 0.339, "_scorer/specialty_derm_std": 0.1009, "_scorer/specialty_derm_n_samples": 14.0, "_scorer/specialty_emergency_score": 0.5588, "_scorer/specialty_emergency_std": 0.1589, "_scorer/specialty_emergency_n_samples": 8.0, "_scorer/specialty_endo_score": 0.4439, "_scorer/specialty_endo_std": 0.1273, "_scorer/specialty_endo_n_samples": 9.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.2, "_scorer/specialty_heme/onc_std": 0.1241, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.1667, "_scorer/specialty_medicine_std": 0.1488, "_scorer/specialty_medicine_n_samples": 6.0, "_scorer/specialty_nephro_score": 0.509, "_scorer/specialty_nephro_std": 0.1374, "_scorer/specialty_nephro_n_samples": 9.0, "_scorer/specialty_neuro_score": 0.4353, "_scorer/specialty_neuro_std": 0.1036, "_scorer/specialty_neuro_n_samples": 15.0, "_scorer/specialty_neurosurg_score": 0.6294, "_scorer/specialty_neurosurg_std": 0.1542, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.3456, "_scorer/specialty_obgyn_std": 0.0907, "_scorer/specialty_obgyn_n_samples": 18.0, "_scorer/specialty_ophtho_score": 0.1511, "_scorer/specialty_ophtho_std": 0.1075, "_scorer/specialty_ophtho_n_samples": 9.0, "_scorer/specialty_ortho_score": 0.3846, "_scorer/specialty_ortho_std": 0.1355, "_scorer/specialty_ortho_n_samples": 13.0, "_scorer/specialty_other_score": 0.1667, "_scorer/specialty_other_std": 0.1536, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_path_score": 0.3333, "_scorer/specialty_path_std": 0.2622, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.1807, "_scorer/specialty_peds_std": 0.0936, "_scorer/specialty_peds_n_samples": 14.0, "_scorer/specialty_primary_score": 0.6176, "_scorer/specialty_primary_std": 0.2129, "_scorer/specialty_primary_n_samples": 4.0, "_scorer/specialty_psychiatry_score": 0.5006, "_scorer/specialty_psychiatry_std": 0.1437, "_scorer/specialty_psychiatry_n_samples": 7.0, "_scorer/specialty_pulm_score": 0.1562, "_scorer/specialty_pulm_std": 0.0936, "_scorer/specialty_pulm_n_samples": 6.0, "_scorer/specialty_rads_score": 0.2143, "_scorer/specialty_rads_std": 0.1346, "_scorer/specialty_rads_n_samples": 7.0, "_scorer/specialty_rheum_score": 0.3762, "_scorer/specialty_rheum_std": 0.1664, "_scorer/specialty_rheum_n_samples": 5.0, "_scorer/specialty_surgery_score": 0.35, "_scorer/specialty_surgery_std": 0.144, "_scorer/specialty_surgery_n_samples": 10.0, "_scorer/specialty_uro_score": 0.5388, "_scorer/specialty_uro_std": 0.1826, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_good_faith_typical_score": 0.4979, "_scorer/source_slice_good_faith_typical_std": 0.0415, "_scorer/source_slice_good_faith_typical_n_samples": 84.0, "_scorer/source_slice_good_faith_difficult_score": 0.2305, "_scorer/source_slice_good_faith_difficult_std": 0.0522, "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2446, "_scorer/source_slice_red_teaming_difficult_std": 0.0435, "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, "healthbench_professional_score/mean": 0.2184, "healthbench_professional_score_length_adjusted/mean": 0.1536 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T19-33-35-00-00_healthbench-professional-writing_eAmnviGLop8QfjxGLMp9Dz.eval", "task": "healthbench_professional_writing", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 142, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 56, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T19:33:35+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1136, "total_samples": 1136, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0147, "_scorer/criteria_met_rate": 0.3924, "_scorer/use_case_writing_score": 0.1977, "_scorer/use_case_writing_std": 0.0287, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_red_teaming_score": 0.0495, "_scorer/type_red_teaming_std": 0.0226, "_scorer/type_red_teaming_n_samples": 91.0, "_scorer/type_good_faith_score": 0.4623, "_scorer/type_good_faith_std": 0.0573, "_scorer/type_good_faith_n_samples": 51.0, "_scorer/difficulty_difficult_score": 0.048, "_scorer/difficulty_difficult_std": 0.0217, "_scorer/difficulty_difficult_n_samples": 96.0, "_scorer/difficulty_typical_score": 0.5101, "_scorer/difficulty_typical_std": 0.0578, "_scorer/difficulty_typical_n_samples": 46.0, "_scorer/specialty_ent_score": 0.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 2.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 5.0, "_scorer/specialty_id_score": 0.0764, "_scorer/specialty_id_std": 0.0581, "_scorer/specialty_id_n_samples": 8.0, "_scorer/specialty_anesthesia_score": 0.2545, "_scorer/specialty_anesthesia_std": 0.1274, "_scorer/specialty_anesthesia_n_samples": 4.0, "_scorer/specialty_cards_score": 0.524, "_scorer/specialty_cards_std": 0.1529, "_scorer/specialty_cards_n_samples": 6.0, "_scorer/specialty_derm_score": 0.0929, "_scorer/specialty_derm_std": 0.0846, "_scorer/specialty_derm_n_samples": 7.0, "_scorer/specialty_emergency_score": 0.1429, "_scorer/specialty_emergency_std": 0.1351, "_scorer/specialty_emergency_n_samples": 7.0, "_scorer/specialty_endo_score": 0.1804, "_scorer/specialty_endo_std": 0.1028, "_scorer/specialty_endo_n_samples": 5.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.263, "_scorer/specialty_heme/onc_std": 0.1242, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.0, "_scorer/specialty_medicine_std": 0.0, "_scorer/specialty_medicine_n_samples": 3.0, "_scorer/specialty_nephro_score": 0.2371, "_scorer/specialty_nephro_std": 0.1017, "_scorer/specialty_nephro_n_samples": 6.0, "_scorer/specialty_neuro_score": 0.1649, "_scorer/specialty_neuro_std": 0.1124, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.25, "_scorer/specialty_neurosurg_std": 0.2179, "_scorer/specialty_neurosurg_n_samples": 4.0, "_scorer/specialty_obgyn_score": 0.25, "_scorer/specialty_obgyn_std": 0.1256, "_scorer/specialty_obgyn_n_samples": 12.0, "_scorer/specialty_ophtho_score": 0.5333, "_scorer/specialty_ophtho_std": 0.1985, "_scorer/specialty_ophtho_n_samples": 5.0, "_scorer/specialty_ortho_score": 0.0, "_scorer/specialty_ortho_std": 0.0, "_scorer/specialty_ortho_n_samples": 4.0, "_scorer/specialty_other_score": 0.1667, "_scorer/specialty_other_std": 0.1175, "_scorer/specialty_other_n_samples": 8.0, "_scorer/specialty_peds_score": 0.25, "_scorer/specialty_peds_std": 0.1238, "_scorer/specialty_peds_n_samples": 8.0, "_scorer/specialty_primary_score": 0.3333, "_scorer/specialty_primary_std": 0.2731, "_scorer/specialty_primary_n_samples": 3.0, "_scorer/specialty_psychiatry_score": 0.1846, "_scorer/specialty_psychiatry_std": 0.099, "_scorer/specialty_psychiatry_n_samples": 12.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 2.0, "_scorer/specialty_rads_score": 0.0, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 1.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 2.0, "_scorer/specialty_surgery_score": 0.25, "_scorer/specialty_surgery_std": 0.2172, "_scorer/specialty_surgery_n_samples": 4.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_red_teaming_difficult_score": 0.0495, "_scorer/source_slice_red_teaming_difficult_std": 0.0225, "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, "_scorer/source_slice_good_faith_difficult_score": 0.0222, "_scorer/source_slice_good_faith_difficult_std": 0.0203, "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, "_scorer/source_slice_good_faith_typical_score": 0.5101, "_scorer/source_slice_good_faith_typical_std": 0.059, "_scorer/source_slice_good_faith_typical_n_samples": 46.0, "healthbench_professional_score/mean": -0.0276, "healthbench_professional_score_length_adjusted/mean": -0.0431 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T19-33-52-00-00_healthbench-professional-research_WF5VpCWo3K285bwUiVdCUm.eval", "task": "healthbench_professional_research", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 147, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 58, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T19:33:52+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1176, "total_samples": 1176, "metrics": { "_scorer/bootstrap_score": 0.2858, "_scorer/std": 0.036, "_scorer/criteria_met_rate": 0.4722, "_scorer/use_case_research_score": 0.4886, "_scorer/use_case_research_std": 0.0344, "_scorer/use_case_research_n_samples": 147.0, "_scorer/type_red_teaming_score": 0.7143, "_scorer/type_red_teaming_std": 0.1786, "_scorer/type_red_teaming_n_samples": 7.0, "_scorer/type_good_faith_score": 0.4773, "_scorer/type_good_faith_std": 0.0344, "_scorer/type_good_faith_n_samples": 140.0, "_scorer/difficulty_difficult_score": 0.4762, "_scorer/difficulty_difficult_std": 0.1095, "_scorer/difficulty_difficult_n_samples": 21.0, "_scorer/difficulty_typical_score": 0.4907, "_scorer/difficulty_typical_std": 0.0348, "_scorer/difficulty_typical_n_samples": 126.0, "_scorer/specialty_ent_score": 1.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 1.0, "_scorer/specialty_gi_score": 0.481, "_scorer/specialty_gi_std": 0.1484, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.7718, "_scorer/specialty_id_std": 0.0991, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_anesthesia_score": 0.5089, "_scorer/specialty_anesthesia_std": 0.1538, "_scorer/specialty_anesthesia_n_samples": 8.0, "_scorer/specialty_cards_score": 0.4794, "_scorer/specialty_cards_std": 0.0889, "_scorer/specialty_cards_n_samples": 14.0, "_scorer/specialty_derm_score": 0.5, "_scorer/specialty_derm_std": 0.2454, "_scorer/specialty_derm_n_samples": 4.0, "_scorer/specialty_emergency_score": 1.0, "_scorer/specialty_emergency_std": 0.0, "_scorer/specialty_emergency_n_samples": 1.0, "_scorer/specialty_endo_score": 0.8867, "_scorer/specialty_endo_std": 0.1064, "_scorer/specialty_endo_n_samples": 6.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 2.0, "_scorer/specialty_heme/onc_score": 0.2597, "_scorer/specialty_heme/onc_std": 0.1351, "_scorer/specialty_heme/onc_n_samples": 7.0, "_scorer/specialty_medicine_score": 0.5043, "_scorer/specialty_medicine_std": 0.1589, "_scorer/specialty_medicine_n_samples": 8.0, "_scorer/specialty_nephro_score": 0.5323, "_scorer/specialty_nephro_std": 0.1498, "_scorer/specialty_nephro_n_samples": 7.0, "_scorer/specialty_neuro_score": 0.4841, "_scorer/specialty_neuro_std": 0.0912, "_scorer/specialty_neuro_n_samples": 19.0, "_scorer/specialty_neurosurg_score": 0.0, "_scorer/specialty_neurosurg_std": 0.0, "_scorer/specialty_neurosurg_n_samples": 1.0, "_scorer/specialty_obgyn_score": 0.4333, "_scorer/specialty_obgyn_std": 0.144, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.75, "_scorer/specialty_ophtho_std": 0.2117, "_scorer/specialty_ophtho_n_samples": 4.0, "_scorer/specialty_ortho_score": 0.4238, "_scorer/specialty_ortho_std": 0.1561, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.3538, "_scorer/specialty_other_std": 0.115, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_peds_score": 0.507, "_scorer/specialty_peds_std": 0.1691, "_scorer/specialty_peds_n_samples": 6.0, "_scorer/specialty_psychiatry_score": 0.1877, "_scorer/specialty_psychiatry_std": 0.1041, "_scorer/specialty_psychiatry_n_samples": 10.0, "_scorer/specialty_pulm_score": 0.4224, "_scorer/specialty_pulm_std": 0.2232, "_scorer/specialty_pulm_n_samples": 4.0, "_scorer/specialty_rheum_score": 1.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 1.0, "_scorer/specialty_surgery_score": 0.3222, "_scorer/specialty_surgery_std": 0.1326, "_scorer/specialty_surgery_n_samples": 3.0, "_scorer/specialty_uro_score": 0.3894, "_scorer/specialty_uro_std": 0.1587, "_scorer/specialty_uro_n_samples": 5.0, "_scorer/source_slice_red_teaming_difficult_score": 0.7143, "_scorer/source_slice_red_teaming_difficult_std": 0.172, "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, "_scorer/source_slice_good_faith_typical_score": 0.4907, "_scorer/source_slice_good_faith_typical_std": 0.0366, "_scorer/source_slice_good_faith_typical_n_samples": 126.0, "_scorer/source_slice_good_faith_difficult_score": 0.3571, "_scorer/source_slice_good_faith_difficult_std": 0.1272, "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, "healthbench_professional_score/mean": 0.4183, "healthbench_professional_score_length_adjusted/mean": 0.2858 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T19-34-11-00-00_healthbench-professional-red-teaming_4db2cTWDv6VCVoFSFqWpmW.eval", "task": "healthbench_professional_red_teaming", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 191, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 76, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T19:34:11+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1528, "total_samples": 1528, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0005, "_scorer/criteria_met_rate": 0.3504, "_scorer/use_case_consult_score": 0.2446, "_scorer/use_case_consult_std": 0.0417, "_scorer/use_case_consult_n_samples": 93.0, "_scorer/use_case_research_score": 0.7143, "_scorer/use_case_research_std": 0.1669, "_scorer/use_case_research_n_samples": 7.0, "_scorer/use_case_writing_score": 0.0495, "_scorer/use_case_writing_std": 0.0224, "_scorer/use_case_writing_n_samples": 91.0, "_scorer/type_red_teaming_score": 0.1688, "_scorer/type_red_teaming_std": 0.0263, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_difficult_score": 0.1688, "_scorer/difficulty_difficult_std": 0.0256, "_scorer/difficulty_difficult_n_samples": 191.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.2499, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 8.0, "_scorer/specialty_id_score": 0.1, "_scorer/specialty_id_std": 0.0608, "_scorer/specialty_id_n_samples": 10.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 1.0, "_scorer/specialty_anesthesia_score": 0.1845, "_scorer/specialty_anesthesia_std": 0.1331, "_scorer/specialty_anesthesia_n_samples": 7.0, "_scorer/specialty_cards_score": 0.423, "_scorer/specialty_cards_std": 0.1408, "_scorer/specialty_cards_n_samples": 11.0, "_scorer/specialty_derm_score": 0.0481, "_scorer/specialty_derm_std": 0.045, "_scorer/specialty_derm_n_samples": 11.0, "_scorer/specialty_emergency_score": 0.3636, "_scorer/specialty_emergency_std": 0.1472, "_scorer/specialty_emergency_n_samples": 11.0, "_scorer/specialty_endo_score": 0.3429, "_scorer/specialty_endo_std": 0.1422, "_scorer/specialty_endo_n_samples": 10.0, "_scorer/specialty_genetics_score": 0.6667, "_scorer/specialty_genetics_std": 0.2708, "_scorer/specialty_genetics_n_samples": 3.0, "_scorer/specialty_heme/onc_score": 0.0, "_scorer/specialty_heme/onc_std": 0.0, "_scorer/specialty_heme/onc_n_samples": 11.0, "_scorer/specialty_medicine_score": 0.1429, "_scorer/specialty_medicine_std": 0.1296, "_scorer/specialty_medicine_n_samples": 7.0, "_scorer/specialty_nephro_score": 0.25, "_scorer/specialty_nephro_std": 0.2105, "_scorer/specialty_nephro_n_samples": 4.0, "_scorer/specialty_neuro_score": 0.2267, "_scorer/specialty_neuro_std": 0.1397, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.2961, "_scorer/specialty_neurosurg_std": 0.1449, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.1, "_scorer/specialty_obgyn_std": 0.0942, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 3.0, "_scorer/specialty_ortho_score": 0.1429, "_scorer/specialty_ortho_std": 0.1272, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.0, "_scorer/specialty_other_std": 0.0, "_scorer/specialty_other_n_samples": 9.0, "_scorer/specialty_path_score": 1.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 1.0, "_scorer/specialty_peds_score": 0.0529, "_scorer/specialty_peds_std": 0.0497, "_scorer/specialty_peds_n_samples": 10.0, "_scorer/specialty_primary_score": 0.4, "_scorer/specialty_primary_std": 0.2181, "_scorer/specialty_primary_n_samples": 5.0, "_scorer/specialty_psychiatry_score": 0.1111, "_scorer/specialty_psychiatry_std": 0.1049, "_scorer/specialty_psychiatry_n_samples": 9.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 5.0, "_scorer/specialty_rads_score": 0.0, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 5.0, "_scorer/specialty_rheum_score": 0.25, "_scorer/specialty_rheum_std": 0.22, "_scorer/specialty_rheum_n_samples": 4.0, "_scorer/specialty_surgery_score": 0.0, "_scorer/specialty_surgery_std": 0.0, "_scorer/specialty_surgery_n_samples": 7.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 3.0, "_scorer/source_slice_red_teaming_difficult_score": 0.1688, "_scorer/source_slice_red_teaming_difficult_std": 0.0252, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": -0.0986, "healthbench_professional_score_length_adjusted/mean": -0.1182 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T19-34-35-00-00_healthbench-professional-physician-baseline_S4NuFZgLXeK7RAgHpRKqTy.eval", "task": "healthbench_professional_physician_baseline", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openai-api/plamo/plamo-3.0-prime", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T19:34:35+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4387, "_scorer/std": 0.0237, "_scorer/criteria_met_rate": 0.5044, "_scorer/use_case_consult_score": 0.4843, "_scorer/use_case_consult_std": 0.0294, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.59, "_scorer/use_case_research_std": 0.0301, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.449, "_scorer/use_case_writing_std": 0.0351, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5561, "_scorer/type_good_faith_std": 0.0208, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4137, "_scorer/type_red_teaming_std": 0.0335, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5803, "_scorer/difficulty_typical_std": 0.0224, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.432, "_scorer/difficulty_difficult_std": 0.0284, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.1752, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.5427, "_scorer/specialty_gi_std": 0.0948, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.5113, "_scorer/specialty_id_std": 0.072, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6229, "_scorer/specialty_anesthesia_std": 0.0868, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4092, "_scorer/specialty_cards_std": 0.0613, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6483, "_scorer/specialty_derm_std": 0.0795, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4549, "_scorer/specialty_emergency_std": 0.1181, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5268, "_scorer/specialty_endo_std": 0.0848, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.7955, "_scorer/specialty_genetics_std": 0.1043, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.5019, "_scorer/specialty_heme/onc_std": 0.0828, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.6713, "_scorer/specialty_medicine_std": 0.0961, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4311, "_scorer/specialty_nephro_std": 0.0806, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5592, "_scorer/specialty_neuro_std": 0.059, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.3242, "_scorer/specialty_neurosurg_std": 0.1149, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4791, "_scorer/specialty_obgyn_std": 0.0644, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3167, "_scorer/specialty_ophtho_std": 0.1, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.3507, "_scorer/specialty_ortho_std": 0.0866, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3541, "_scorer/specialty_other_std": 0.0879, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.6667, "_scorer/specialty_path_std": 0.2675, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.5363, "_scorer/specialty_peds_std": 0.0829, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1577, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5115, "_scorer/specialty_psychiatry_std": 0.074, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5848, "_scorer/specialty_pulm_std": 0.1155, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.3787, "_scorer/specialty_rads_std": 0.1435, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.6964, "_scorer/specialty_rheum_std": 0.1477, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.498, "_scorer/specialty_surgery_std": 0.1199, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6175, "_scorer/specialty_uro_std": 0.1167, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5803, "_scorer/source_slice_good_faith_typical_std": 0.0225, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.4768, "_scorer/source_slice_good_faith_difficult_std": 0.0528, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4137, "_scorer/source_slice_red_teaming_difficult_std": 0.0321, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.4429, "healthbench_professional_score_length_adjusted/mean": 0.4387 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "physician_baseline_solver" }, { "space": "hs-prof-subsets", "file": "2026-07-24T19-59-01-00-00_healthbench-professional-writing_933975rUF6cdWKJECk44bJ.eval", "task": "healthbench_professional_writing", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 142, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 56, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T19:59:01+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1136, "total_samples": 1136, "metrics": { "_scorer/bootstrap_score": 0.0495, "_scorer/std": 0.0457, "_scorer/criteria_met_rate": 0.5576, "_scorer/use_case_writing_score": 0.346, "_scorer/use_case_writing_std": 0.0362, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_red_teaming_score": 0.1183, "_scorer/type_red_teaming_std": 0.032, "_scorer/type_red_teaming_n_samples": 91.0, "_scorer/type_good_faith_score": 0.7522, "_scorer/type_good_faith_std": 0.0469, "_scorer/type_good_faith_n_samples": 51.0, "_scorer/difficulty_difficult_score": 0.1486, "_scorer/difficulty_difficult_std": 0.0312, "_scorer/difficulty_difficult_n_samples": 96.0, "_scorer/difficulty_typical_score": 0.7579, "_scorer/difficulty_typical_std": 0.046, "_scorer/difficulty_typical_n_samples": 46.0, "_scorer/specialty_ent_score": 0.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 2.0, "_scorer/specialty_gi_score": 0.2, "_scorer/specialty_gi_std": 0.1788, "_scorer/specialty_gi_n_samples": 5.0, "_scorer/specialty_id_score": 0.275, "_scorer/specialty_id_std": 0.1219, "_scorer/specialty_id_n_samples": 8.0, "_scorer/specialty_anesthesia_score": 0.4318, "_scorer/specialty_anesthesia_std": 0.2228, "_scorer/specialty_anesthesia_n_samples": 4.0, "_scorer/specialty_cards_score": 0.5694, "_scorer/specialty_cards_std": 0.1709, "_scorer/specialty_cards_n_samples": 6.0, "_scorer/specialty_derm_score": 0.1429, "_scorer/specialty_derm_std": 0.1317, "_scorer/specialty_derm_n_samples": 7.0, "_scorer/specialty_emergency_score": 0.1429, "_scorer/specialty_emergency_std": 0.133, "_scorer/specialty_emergency_n_samples": 7.0, "_scorer/specialty_endo_score": 0.3474, "_scorer/specialty_endo_std": 0.187, "_scorer/specialty_endo_n_samples": 5.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.277, "_scorer/specialty_heme/onc_std": 0.13, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.3333, "_scorer/specialty_medicine_std": 0.2718, "_scorer/specialty_medicine_n_samples": 3.0, "_scorer/specialty_nephro_score": 0.3359, "_scorer/specialty_nephro_std": 0.1558, "_scorer/specialty_nephro_n_samples": 6.0, "_scorer/specialty_neuro_score": 0.4409, "_scorer/specialty_neuro_std": 0.1469, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.375, "_scorer/specialty_neurosurg_std": 0.2108, "_scorer/specialty_neurosurg_n_samples": 4.0, "_scorer/specialty_obgyn_score": 0.5491, "_scorer/specialty_obgyn_std": 0.1146, "_scorer/specialty_obgyn_n_samples": 12.0, "_scorer/specialty_ophtho_score": 0.475, "_scorer/specialty_ophtho_std": 0.2042, "_scorer/specialty_ophtho_n_samples": 5.0, "_scorer/specialty_ortho_score": 0.25, "_scorer/specialty_ortho_std": 0.2124, "_scorer/specialty_ortho_n_samples": 4.0, "_scorer/specialty_other_score": 0.2917, "_scorer/specialty_other_std": 0.1461, "_scorer/specialty_other_n_samples": 8.0, "_scorer/specialty_peds_score": 0.625, "_scorer/specialty_peds_std": 0.1446, "_scorer/specialty_peds_n_samples": 8.0, "_scorer/specialty_primary_score": 0.3333, "_scorer/specialty_primary_std": 0.2746, "_scorer/specialty_primary_n_samples": 3.0, "_scorer/specialty_psychiatry_score": 0.5028, "_scorer/specialty_psychiatry_std": 0.1284, "_scorer/specialty_psychiatry_n_samples": 12.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 2.0, "_scorer/specialty_rads_score": 0.4667, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 1.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 2.0, "_scorer/specialty_surgery_score": 0.25, "_scorer/specialty_surgery_std": 0.2225, "_scorer/specialty_surgery_n_samples": 4.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_red_teaming_difficult_score": 0.1183, "_scorer/source_slice_red_teaming_difficult_std": 0.0305, "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, "_scorer/source_slice_good_faith_difficult_score": 0.7, "_scorer/source_slice_good_faith_difficult_std": 0.1781, "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, "_scorer/source_slice_good_faith_typical_score": 0.7579, "_scorer/source_slice_good_faith_typical_std": 0.0467, "_scorer/source_slice_good_faith_typical_n_samples": 46.0, "healthbench_professional_score/mean": 0.0954, "healthbench_professional_score_length_adjusted/mean": 0.0495 }, "model_usage_models": [ "openai/gpt-5.4", "openrouter/deepseek/deepseek-v4-pro" ], "model_usage_total": { "openai/gpt-5.4": 3316215, "openrouter/deepseek/deepseek-v4-pro": 1479207 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T20-07-30-00-00_healthbench-professional-research_7gKGNrErBwsumd7oKKUcuN.eval", "task": "healthbench_professional_research", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 147, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 58, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T20:07:30+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1176, "total_samples": 1176, "metrics": { "_scorer/bootstrap_score": 0.5291, "_scorer/std": 0.0307, "_scorer/criteria_met_rate": 0.6757, "_scorer/use_case_research_score": 0.6937, "_scorer/use_case_research_std": 0.0317, "_scorer/use_case_research_n_samples": 147.0, "_scorer/type_red_teaming_score": 0.8571, "_scorer/type_red_teaming_std": 0.1272, "_scorer/type_red_teaming_n_samples": 7.0, "_scorer/type_good_faith_score": 0.6855, "_scorer/type_good_faith_std": 0.0319, "_scorer/type_good_faith_n_samples": 140.0, "_scorer/difficulty_difficult_score": 0.5952, "_scorer/difficulty_difficult_std": 0.1028, "_scorer/difficulty_difficult_n_samples": 21.0, "_scorer/difficulty_typical_score": 0.7101, "_scorer/difficulty_typical_std": 0.0304, "_scorer/difficulty_typical_n_samples": 126.0, "_scorer/specialty_ent_score": 1.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 1.0, "_scorer/specialty_gi_score": 0.6869, "_scorer/specialty_gi_std": 0.1124, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.7302, "_scorer/specialty_id_std": 0.0839, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_anesthesia_score": 0.8214, "_scorer/specialty_anesthesia_std": 0.0811, "_scorer/specialty_anesthesia_n_samples": 8.0, "_scorer/specialty_cards_score": 0.7352, "_scorer/specialty_cards_std": 0.0923, "_scorer/specialty_cards_n_samples": 14.0, "_scorer/specialty_derm_score": 1.0, "_scorer/specialty_derm_std": 0.0, "_scorer/specialty_derm_n_samples": 4.0, "_scorer/specialty_emergency_score": 0.0, "_scorer/specialty_emergency_std": 0.0, "_scorer/specialty_emergency_n_samples": 1.0, "_scorer/specialty_endo_score": 0.8567, "_scorer/specialty_endo_std": 0.0856, "_scorer/specialty_endo_n_samples": 6.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 2.0, "_scorer/specialty_heme/onc_score": 0.6926, "_scorer/specialty_heme/onc_std": 0.1316, "_scorer/specialty_heme/onc_n_samples": 7.0, "_scorer/specialty_medicine_score": 0.427, "_scorer/specialty_medicine_std": 0.155, "_scorer/specialty_medicine_n_samples": 8.0, "_scorer/specialty_nephro_score": 0.8538, "_scorer/specialty_nephro_std": 0.0906, "_scorer/specialty_nephro_n_samples": 7.0, "_scorer/specialty_neuro_score": 0.672, "_scorer/specialty_neuro_std": 0.0899, "_scorer/specialty_neuro_n_samples": 19.0, "_scorer/specialty_neurosurg_score": 0.0667, "_scorer/specialty_neurosurg_std": 0.0, "_scorer/specialty_neurosurg_n_samples": 1.0, "_scorer/specialty_obgyn_score": 0.4816, "_scorer/specialty_obgyn_std": 0.1137, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 1.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 4.0, "_scorer/specialty_ortho_score": 0.2988, "_scorer/specialty_ortho_std": 0.1309, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.6736, "_scorer/specialty_other_std": 0.1413, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_peds_score": 0.7143, "_scorer/specialty_peds_std": 0.1645, "_scorer/specialty_peds_n_samples": 6.0, "_scorer/specialty_psychiatry_score": 0.725, "_scorer/specialty_psychiatry_std": 0.1327, "_scorer/specialty_psychiatry_n_samples": 10.0, "_scorer/specialty_pulm_score": 0.6026, "_scorer/specialty_pulm_std": 0.1386, "_scorer/specialty_pulm_n_samples": 4.0, "_scorer/specialty_rheum_score": 1.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 1.0, "_scorer/specialty_surgery_score": 1.0, "_scorer/specialty_surgery_std": 0.0, "_scorer/specialty_surgery_n_samples": 3.0, "_scorer/specialty_uro_score": 0.7609, "_scorer/specialty_uro_std": 0.1327, "_scorer/specialty_uro_n_samples": 5.0, "_scorer/source_slice_red_teaming_difficult_score": 0.8571, "_scorer/source_slice_red_teaming_difficult_std": 0.1309, "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, "_scorer/source_slice_good_faith_typical_score": 0.7101, "_scorer/source_slice_good_faith_typical_std": 0.0315, "_scorer/source_slice_good_faith_typical_n_samples": 126.0, "_scorer/source_slice_good_faith_difficult_score": 0.4643, "_scorer/source_slice_good_faith_difficult_std": 0.1296, "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, "healthbench_professional_score/mean": 0.6368, "healthbench_professional_score_length_adjusted/mean": 0.5291 }, "model_usage_models": [ "openai/gpt-5.4", "openrouter/deepseek/deepseek-v4-pro" ], "model_usage_total": { "openai/gpt-5.4": 4316530, "openrouter/deepseek/deepseek-v4-pro": 2021029 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T20-18-24-00-00_healthbench-professional-red-teaming_HmwUcZnfLNqmUkyZkHDTdJ.eval", "task": "healthbench_professional_red_teaming", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 191, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 76, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T20:18:24+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 1528, "total_samples": 1528, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0072, "_scorer/criteria_met_rate": 0.4732, "_scorer/use_case_consult_score": 0.3379, "_scorer/use_case_consult_std": 0.0446, "_scorer/use_case_consult_n_samples": 93.0, "_scorer/use_case_research_score": 0.8571, "_scorer/use_case_research_std": 0.1282, "_scorer/use_case_research_n_samples": 7.0, "_scorer/use_case_writing_score": 0.1183, "_scorer/use_case_writing_std": 0.0301, "_scorer/use_case_writing_n_samples": 91.0, "_scorer/type_red_teaming_score": 0.2523, "_scorer/type_red_teaming_std": 0.0296, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_difficult_score": 0.2523, "_scorer/difficulty_difficult_std": 0.0302, "_scorer/difficulty_difficult_n_samples": 191.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.2452, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 8.0, "_scorer/specialty_id_score": 0.27, "_scorer/specialty_id_std": 0.1248, "_scorer/specialty_id_n_samples": 10.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 1.0, "_scorer/specialty_anesthesia_score": 0.5238, "_scorer/specialty_anesthesia_std": 0.1752, "_scorer/specialty_anesthesia_n_samples": 7.0, "_scorer/specialty_cards_score": 0.1818, "_scorer/specialty_cards_std": 0.1151, "_scorer/specialty_cards_n_samples": 11.0, "_scorer/specialty_derm_score": 0.0481, "_scorer/specialty_derm_std": 0.0469, "_scorer/specialty_derm_n_samples": 11.0, "_scorer/specialty_emergency_score": 0.2727, "_scorer/specialty_emergency_std": 0.1308, "_scorer/specialty_emergency_n_samples": 11.0, "_scorer/specialty_endo_score": 0.3429, "_scorer/specialty_endo_std": 0.1394, "_scorer/specialty_endo_n_samples": 10.0, "_scorer/specialty_genetics_score": 0.5, "_scorer/specialty_genetics_std": 0.2264, "_scorer/specialty_genetics_n_samples": 3.0, "_scorer/specialty_heme/onc_score": 0.1316, "_scorer/specialty_heme/onc_std": 0.0619, "_scorer/specialty_heme/onc_n_samples": 11.0, "_scorer/specialty_medicine_score": 0.1429, "_scorer/specialty_medicine_std": 0.1292, "_scorer/specialty_medicine_n_samples": 7.0, "_scorer/specialty_nephro_score": 0.5, "_scorer/specialty_nephro_std": 0.2551, "_scorer/specialty_nephro_n_samples": 4.0, "_scorer/specialty_neuro_score": 0.2622, "_scorer/specialty_neuro_std": 0.1331, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.4762, "_scorer/specialty_neurosurg_std": 0.17, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.4128, "_scorer/specialty_obgyn_std": 0.122, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.3333, "_scorer/specialty_ophtho_std": 0.2694, "_scorer/specialty_ophtho_n_samples": 3.0, "_scorer/specialty_ortho_score": 0.4286, "_scorer/specialty_ortho_std": 0.1876, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.1111, "_scorer/specialty_other_std": 0.1025, "_scorer/specialty_other_n_samples": 9.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 1.0, "_scorer/specialty_peds_score": 0.2111, "_scorer/specialty_peds_std": 0.1248, "_scorer/specialty_peds_n_samples": 10.0, "_scorer/specialty_primary_score": 0.4, "_scorer/specialty_primary_std": 0.2194, "_scorer/specialty_primary_n_samples": 5.0, "_scorer/specialty_psychiatry_score": 0.2222, "_scorer/specialty_psychiatry_std": 0.1348, "_scorer/specialty_psychiatry_n_samples": 9.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 5.0, "_scorer/specialty_rads_score": 0.4933, "_scorer/specialty_rads_std": 0.1963, "_scorer/specialty_rads_n_samples": 5.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 4.0, "_scorer/specialty_surgery_score": 0.2857, "_scorer/specialty_surgery_std": 0.1707, "_scorer/specialty_surgery_n_samples": 7.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 3.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2523, "_scorer/source_slice_red_teaming_difficult_std": 0.0304, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": -0.0372, "healthbench_professional_score_length_adjusted/mean": -0.0687 }, "model_usage_models": [ "openai/gpt-5.4", "openrouter/deepseek/deepseek-v4-pro" ], "model_usage_total": { "openai/gpt-5.4": 2312783, "openrouter/deepseek/deepseek-v4-pro": 1062156 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T20-28-47-00-00_healthbench-professional-physician-baseline_kjY8x2CXBcbHZUSHs55G7D.eval", "task": "healthbench_professional_physician_baseline", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T20:28:47+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4387, "_scorer/std": 0.0241, "_scorer/criteria_met_rate": 0.5044, "_scorer/use_case_consult_score": 0.4843, "_scorer/use_case_consult_std": 0.0285, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.59, "_scorer/use_case_research_std": 0.0304, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.449, "_scorer/use_case_writing_std": 0.0366, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5561, "_scorer/type_good_faith_std": 0.0211, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4137, "_scorer/type_red_teaming_std": 0.0314, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5803, "_scorer/difficulty_typical_std": 0.0226, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.432, "_scorer/difficulty_difficult_std": 0.0279, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.1788, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.5427, "_scorer/specialty_gi_std": 0.0926, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.5113, "_scorer/specialty_id_std": 0.0747, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6229, "_scorer/specialty_anesthesia_std": 0.0871, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.4092, "_scorer/specialty_cards_std": 0.0638, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6483, "_scorer/specialty_derm_std": 0.0777, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4549, "_scorer/specialty_emergency_std": 0.1139, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5268, "_scorer/specialty_endo_std": 0.0838, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.7955, "_scorer/specialty_genetics_std": 0.1088, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.5019, "_scorer/specialty_heme/onc_std": 0.0829, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.6713, "_scorer/specialty_medicine_std": 0.097, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4311, "_scorer/specialty_nephro_std": 0.0806, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5592, "_scorer/specialty_neuro_std": 0.059, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.3242, "_scorer/specialty_neurosurg_std": 0.1121, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4791, "_scorer/specialty_obgyn_std": 0.0634, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3167, "_scorer/specialty_ophtho_std": 0.0981, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.3507, "_scorer/specialty_ortho_std": 0.0868, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3541, "_scorer/specialty_other_std": 0.0869, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.6667, "_scorer/specialty_path_std": 0.2605, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.5363, "_scorer/specialty_peds_std": 0.0825, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1593, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5115, "_scorer/specialty_psychiatry_std": 0.0715, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5848, "_scorer/specialty_pulm_std": 0.12, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.3787, "_scorer/specialty_rads_std": 0.1469, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.6964, "_scorer/specialty_rheum_std": 0.1457, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.498, "_scorer/specialty_surgery_std": 0.1162, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6175, "_scorer/specialty_uro_std": 0.1148, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5803, "_scorer/source_slice_good_faith_typical_std": 0.0225, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.4768, "_scorer/source_slice_good_faith_difficult_std": 0.0516, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4137, "_scorer/source_slice_red_teaming_difficult_std": 0.0324, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.4429, "healthbench_professional_score_length_adjusted/mean": 0.4387 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "physician_baseline_solver" }, { "space": "hs-prof-subsets", "file": "2026-07-24T23-41-11-00-00_healthbench-professional-consult_muhCA3DyeXvwNpDvn7gpN4.eval", "task": "healthbench_professional_consult", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 236, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 94, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T23:41:11+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 1888, "total_samples": 1888, "metrics": { "_scorer/bootstrap_score": 0.1778, "_scorer/std": 0.0344, "_scorer/criteria_met_rate": 0.465, "_scorer/use_case_consult_score": 0.3894, "_scorer/use_case_consult_std": 0.0288, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/type_good_faith_score": 0.4788, "_scorer/type_good_faith_std": 0.0361, "_scorer/type_good_faith_n_samples": 143.0, "_scorer/type_red_teaming_score": 0.2521, "_scorer/type_red_teaming_std": 0.042, "_scorer/type_red_teaming_n_samples": 93.0, "_scorer/difficulty_typical_score": 0.5822, "_scorer/difficulty_typical_std": 0.0444, "_scorer/difficulty_typical_n_samples": 84.0, "_scorer/difficulty_difficult_score": 0.2829, "_scorer/difficulty_difficult_std": 0.0349, "_scorer/difficulty_difficult_n_samples": 152.0, "_scorer/specialty_ent_score": 0.8271, "_scorer/specialty_ent_std": 0.0957, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.5251, "_scorer/specialty_id_std": 0.1189, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.3639, "_scorer/specialty_anesthesia_std": 0.1367, "_scorer/specialty_anesthesia_n_samples": 10.0, "_scorer/specialty_cards_score": 0.2994, "_scorer/specialty_cards_std": 0.0859, "_scorer/specialty_cards_n_samples": 23.0, "_scorer/specialty_derm_score": 0.4374, "_scorer/specialty_derm_std": 0.1206, "_scorer/specialty_derm_n_samples": 14.0, "_scorer/specialty_emergency_score": 0.6838, "_scorer/specialty_emergency_std": 0.1467, "_scorer/specialty_emergency_n_samples": 8.0, "_scorer/specialty_endo_score": 0.4092, "_scorer/specialty_endo_std": 0.1105, "_scorer/specialty_endo_n_samples": 9.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.3, "_scorer/specialty_heme/onc_std": 0.1481, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.0778, "_scorer/specialty_medicine_std": 0.0723, "_scorer/specialty_medicine_n_samples": 6.0, "_scorer/specialty_nephro_score": 0.6452, "_scorer/specialty_nephro_std": 0.1375, "_scorer/specialty_nephro_n_samples": 9.0, "_scorer/specialty_neuro_score": 0.4072, "_scorer/specialty_neuro_std": 0.1084, "_scorer/specialty_neuro_n_samples": 15.0, "_scorer/specialty_neurosurg_score": 0.6801, "_scorer/specialty_neurosurg_std": 0.1509, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.3689, "_scorer/specialty_obgyn_std": 0.0948, "_scorer/specialty_obgyn_n_samples": 18.0, "_scorer/specialty_ophtho_score": 0.4089, "_scorer/specialty_ophtho_std": 0.1586, "_scorer/specialty_ophtho_n_samples": 9.0, "_scorer/specialty_ortho_score": 0.506, "_scorer/specialty_ortho_std": 0.1323, "_scorer/specialty_ortho_n_samples": 13.0, "_scorer/specialty_other_score": 0.3333, "_scorer/specialty_other_std": 0.1868, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.2222, "_scorer/specialty_peds_std": 0.1092, "_scorer/specialty_peds_n_samples": 14.0, "_scorer/specialty_primary_score": 0.6176, "_scorer/specialty_primary_std": 0.2088, "_scorer/specialty_primary_n_samples": 4.0, "_scorer/specialty_psychiatry_score": 0.4291, "_scorer/specialty_psychiatry_std": 0.1583, "_scorer/specialty_psychiatry_n_samples": 7.0, "_scorer/specialty_pulm_score": 0.3229, "_scorer/specialty_pulm_std": 0.1575, "_scorer/specialty_pulm_n_samples": 6.0, "_scorer/specialty_rads_score": 0.4286, "_scorer/specialty_rads_std": 0.1857, "_scorer/specialty_rads_n_samples": 7.0, "_scorer/specialty_rheum_score": 0.2524, "_scorer/specialty_rheum_std": 0.1416, "_scorer/specialty_rheum_n_samples": 5.0, "_scorer/specialty_surgery_score": 0.4, "_scorer/specialty_surgery_std": 0.1568, "_scorer/specialty_surgery_n_samples": 10.0, "_scorer/specialty_uro_score": 0.625, "_scorer/specialty_uro_std": 0.2167, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_good_faith_typical_score": 0.5822, "_scorer/source_slice_good_faith_typical_std": 0.0443, "_scorer/source_slice_good_faith_typical_n_samples": 84.0, "_scorer/source_slice_good_faith_difficult_score": 0.3314, "_scorer/source_slice_good_faith_difficult_std": 0.0576, "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2521, "_scorer/source_slice_red_teaming_difficult_std": 0.0414, "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, "healthbench_professional_score/mean": 0.2851, "healthbench_professional_score_length_adjusted/mean": 0.1778 }, "model_usage_models": [ "vllm/google/medgemma-27b-text-it", "openai/gpt-5.4" ], "model_usage_total": { "vllm/google/medgemma-27b-text-it": 23572, "openai/gpt-5.4": 61467 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T23-41-59-00-00_healthbench-professional-writing_CMBT9aSmchzcQFRpwZdCqR.eval", "task": "healthbench_professional_writing", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 142, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 56, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T23:41:59+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 1136, "total_samples": 1136, "metrics": { "_scorer/bootstrap_score": 0.0913, "_scorer/std": 0.0528, "_scorer/criteria_met_rate": 0.5106, "_scorer/use_case_writing_score": 0.3879, "_scorer/use_case_writing_std": 0.0371, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_red_teaming_score": 0.2182, "_scorer/type_red_teaming_std": 0.04, "_scorer/type_red_teaming_n_samples": 91.0, "_scorer/type_good_faith_score": 0.6907, "_scorer/type_good_faith_std": 0.0478, "_scorer/type_good_faith_n_samples": 51.0, "_scorer/difficulty_difficult_score": 0.2225, "_scorer/difficulty_difficult_std": 0.0394, "_scorer/difficulty_difficult_n_samples": 96.0, "_scorer/difficulty_typical_score": 0.7332, "_scorer/difficulty_typical_std": 0.0447, "_scorer/difficulty_typical_n_samples": 46.0, "_scorer/specialty_ent_score": 0.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 2.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 5.0, "_scorer/specialty_id_score": 0.25, "_scorer/specialty_id_std": 0.1291, "_scorer/specialty_id_n_samples": 8.0, "_scorer/specialty_anesthesia_score": 0.4318, "_scorer/specialty_anesthesia_std": 0.2177, "_scorer/specialty_anesthesia_n_samples": 4.0, "_scorer/specialty_cards_score": 0.4634, "_scorer/specialty_cards_std": 0.1538, "_scorer/specialty_cards_n_samples": 6.0, "_scorer/specialty_derm_score": 0.2357, "_scorer/specialty_derm_std": 0.1489, "_scorer/specialty_derm_n_samples": 7.0, "_scorer/specialty_emergency_score": 0.3661, "_scorer/specialty_emergency_std": 0.1672, "_scorer/specialty_emergency_n_samples": 7.0, "_scorer/specialty_endo_score": 0.254, "_scorer/specialty_endo_std": 0.1405, "_scorer/specialty_endo_n_samples": 5.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.477, "_scorer/specialty_heme/onc_std": 0.1453, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.3333, "_scorer/specialty_medicine_std": 0.2691, "_scorer/specialty_medicine_n_samples": 3.0, "_scorer/specialty_nephro_score": 0.6275, "_scorer/specialty_nephro_std": 0.1494, "_scorer/specialty_nephro_n_samples": 6.0, "_scorer/specialty_neuro_score": 0.3602, "_scorer/specialty_neuro_std": 0.1432, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.375, "_scorer/specialty_neurosurg_std": 0.2098, "_scorer/specialty_neurosurg_n_samples": 4.0, "_scorer/specialty_obgyn_score": 0.5261, "_scorer/specialty_obgyn_std": 0.1317, "_scorer/specialty_obgyn_n_samples": 12.0, "_scorer/specialty_ophtho_score": 0.5333, "_scorer/specialty_ophtho_std": 0.1997, "_scorer/specialty_ophtho_n_samples": 5.0, "_scorer/specialty_ortho_score": 0.5, "_scorer/specialty_ortho_std": 0.246, "_scorer/specialty_ortho_n_samples": 4.0, "_scorer/specialty_other_score": 0.2917, "_scorer/specialty_other_std": 0.1454, "_scorer/specialty_other_n_samples": 8.0, "_scorer/specialty_peds_score": 0.625, "_scorer/specialty_peds_std": 0.1471, "_scorer/specialty_peds_n_samples": 8.0, "_scorer/specialty_primary_score": 0.4902, "_scorer/specialty_primary_std": 0.2381, "_scorer/specialty_primary_n_samples": 3.0, "_scorer/specialty_psychiatry_score": 0.5028, "_scorer/specialty_psychiatry_std": 0.1346, "_scorer/specialty_psychiatry_n_samples": 12.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 2.0, "_scorer/specialty_rads_score": 0.0, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 1.0, "_scorer/specialty_rheum_score": 0.5, "_scorer/specialty_rheum_std": 0.3484, "_scorer/specialty_rheum_n_samples": 2.0, "_scorer/specialty_surgery_score": 0.25, "_scorer/specialty_surgery_std": 0.2285, "_scorer/specialty_surgery_n_samples": 4.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2182, "_scorer/source_slice_red_teaming_difficult_std": 0.0412, "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, "_scorer/source_slice_good_faith_difficult_score": 0.3, "_scorer/source_slice_good_faith_difficult_std": 0.1763, "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, "_scorer/source_slice_good_faith_typical_score": 0.7332, "_scorer/source_slice_good_faith_typical_std": 0.0439, "_scorer/source_slice_good_faith_typical_n_samples": 46.0, "healthbench_professional_score/mean": 0.1891, "healthbench_professional_score_length_adjusted/mean": 0.0913 }, "model_usage_models": [ "vllm/google/medgemma-27b-text-it", "openai/gpt-5.4" ], "model_usage_total": { "vllm/google/medgemma-27b-text-it": 5260, "openai/gpt-5.4": 19696 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T23-42-33-00-00_healthbench-professional-research_RFx9qF5ZP369Kdi3CNQU6M.eval", "task": "healthbench_professional_research", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 147, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 58, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T23:42:33+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 1176, "total_samples": 1176, "metrics": { "_scorer/bootstrap_score": 0.344, "_scorer/std": 0.0351, "_scorer/criteria_met_rate": 0.5214, "_scorer/use_case_research_score": 0.5213, "_scorer/use_case_research_std": 0.0339, "_scorer/use_case_research_n_samples": 147.0, "_scorer/type_red_teaming_score": 0.7143, "_scorer/type_red_teaming_std": 0.1773, "_scorer/type_red_teaming_n_samples": 7.0, "_scorer/type_good_faith_score": 0.5116, "_scorer/type_good_faith_std": 0.034, "_scorer/type_good_faith_n_samples": 140.0, "_scorer/difficulty_difficult_score": 0.451, "_scorer/difficulty_difficult_std": 0.1048, "_scorer/difficulty_difficult_n_samples": 21.0, "_scorer/difficulty_typical_score": 0.533, "_scorer/difficulty_typical_std": 0.0344, "_scorer/difficulty_typical_n_samples": 126.0, "_scorer/specialty_ent_score": 1.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 1.0, "_scorer/specialty_gi_score": 0.4415, "_scorer/specialty_gi_std": 0.1493, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.5615, "_scorer/specialty_id_std": 0.1441, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_anesthesia_score": 0.4464, "_scorer/specialty_anesthesia_std": 0.1392, "_scorer/specialty_anesthesia_n_samples": 8.0, "_scorer/specialty_cards_score": 0.5548, "_scorer/specialty_cards_std": 0.0978, "_scorer/specialty_cards_n_samples": 14.0, "_scorer/specialty_derm_score": 1.0, "_scorer/specialty_derm_std": 0.0, "_scorer/specialty_derm_n_samples": 4.0, "_scorer/specialty_emergency_score": 0.0, "_scorer/specialty_emergency_std": 0.0, "_scorer/specialty_emergency_n_samples": 1.0, "_scorer/specialty_endo_score": 0.94, "_scorer/specialty_endo_std": 0.0547, "_scorer/specialty_endo_n_samples": 6.0, "_scorer/specialty_genetics_score": 1.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 2.0, "_scorer/specialty_heme/onc_score": 0.2317, "_scorer/specialty_heme/onc_std": 0.0791, "_scorer/specialty_heme/onc_n_samples": 7.0, "_scorer/specialty_medicine_score": 0.625, "_scorer/specialty_medicine_std": 0.1735, "_scorer/specialty_medicine_n_samples": 8.0, "_scorer/specialty_nephro_score": 0.4507, "_scorer/specialty_nephro_std": 0.1604, "_scorer/specialty_nephro_n_samples": 7.0, "_scorer/specialty_neuro_score": 0.5824, "_scorer/specialty_neuro_std": 0.0874, "_scorer/specialty_neuro_n_samples": 19.0, "_scorer/specialty_neurosurg_score": 0.0, "_scorer/specialty_neurosurg_std": 0.0, "_scorer/specialty_neurosurg_n_samples": 1.0, "_scorer/specialty_obgyn_score": 0.3167, "_scorer/specialty_obgyn_std": 0.1306, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.625, "_scorer/specialty_ophtho_std": 0.2058, "_scorer/specialty_ophtho_n_samples": 4.0, "_scorer/specialty_ortho_score": 0.4238, "_scorer/specialty_ortho_std": 0.1465, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.5712, "_scorer/specialty_other_std": 0.1538, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_peds_score": 0.5003, "_scorer/specialty_peds_std": 0.1704, "_scorer/specialty_peds_n_samples": 6.0, "_scorer/specialty_psychiatry_score": 0.3217, "_scorer/specialty_psychiatry_std": 0.1213, "_scorer/specialty_psychiatry_n_samples": 10.0, "_scorer/specialty_pulm_score": 0.4224, "_scorer/specialty_pulm_std": 0.2187, "_scorer/specialty_pulm_n_samples": 4.0, "_scorer/specialty_rheum_score": 0.5714, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 1.0, "_scorer/specialty_surgery_score": 0.6556, "_scorer/specialty_surgery_std": 0.1419, "_scorer/specialty_surgery_n_samples": 3.0, "_scorer/specialty_uro_score": 0.5675, "_scorer/specialty_uro_std": 0.176, "_scorer/specialty_uro_n_samples": 5.0, "_scorer/source_slice_red_teaming_difficult_score": 0.7143, "_scorer/source_slice_red_teaming_difficult_std": 0.1676, "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, "_scorer/source_slice_good_faith_typical_score": 0.533, "_scorer/source_slice_good_faith_typical_std": 0.0367, "_scorer/source_slice_good_faith_typical_n_samples": 126.0, "_scorer/source_slice_good_faith_difficult_score": 0.3193, "_scorer/source_slice_good_faith_difficult_std": 0.1191, "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, "healthbench_professional_score/mean": 0.4776, "healthbench_professional_score_length_adjusted/mean": 0.344 }, "model_usage_models": [ "vllm/google/medgemma-27b-text-it", "openai/gpt-5.4" ], "model_usage_total": { "vllm/google/medgemma-27b-text-it": 12406, "openai/gpt-5.4": 45185 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T23-43-33-00-00_healthbench-professional-red-teaming_ewG7cuk2vXaZpcauRLz7qR.eval", "task": "healthbench_professional_red_teaming", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 191, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 76, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T23:43:33+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 1528, "total_samples": 1528, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0053, "_scorer/criteria_met_rate": 0.4126, "_scorer/use_case_consult_score": 0.2521, "_scorer/use_case_consult_std": 0.0418, "_scorer/use_case_consult_n_samples": 93.0, "_scorer/use_case_research_score": 0.7143, "_scorer/use_case_research_std": 0.1668, "_scorer/use_case_research_n_samples": 7.0, "_scorer/use_case_writing_score": 0.2182, "_scorer/use_case_writing_std": 0.0421, "_scorer/use_case_writing_n_samples": 91.0, "_scorer/type_red_teaming_score": 0.2529, "_scorer/type_red_teaming_std": 0.0292, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_difficult_score": 0.2529, "_scorer/difficulty_difficult_std": 0.0293, "_scorer/difficulty_difficult_n_samples": 191.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.2522, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 8.0, "_scorer/specialty_id_score": 0.2471, "_scorer/specialty_id_std": 0.1041, "_scorer/specialty_id_n_samples": 10.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 1.0, "_scorer/specialty_anesthesia_score": 0.3274, "_scorer/specialty_anesthesia_std": 0.1669, "_scorer/specialty_anesthesia_n_samples": 7.0, "_scorer/specialty_cards_score": 0.3393, "_scorer/specialty_cards_std": 0.1297, "_scorer/specialty_cards_n_samples": 11.0, "_scorer/specialty_derm_score": 0.1818, "_scorer/specialty_derm_std": 0.1156, "_scorer/specialty_derm_n_samples": 11.0, "_scorer/specialty_emergency_score": 0.4148, "_scorer/specialty_emergency_std": 0.1421, "_scorer/specialty_emergency_n_samples": 11.0, "_scorer/specialty_endo_score": 0.2899, "_scorer/specialty_endo_std": 0.1183, "_scorer/specialty_endo_n_samples": 10.0, "_scorer/specialty_genetics_score": 0.6667, "_scorer/specialty_genetics_std": 0.2711, "_scorer/specialty_genetics_n_samples": 3.0, "_scorer/specialty_heme/onc_score": 0.3158, "_scorer/specialty_heme/onc_std": 0.1289, "_scorer/specialty_heme/onc_n_samples": 11.0, "_scorer/specialty_medicine_score": 0.1429, "_scorer/specialty_medicine_std": 0.1347, "_scorer/specialty_medicine_n_samples": 7.0, "_scorer/specialty_nephro_score": 0.75, "_scorer/specialty_nephro_std": 0.2103, "_scorer/specialty_nephro_n_samples": 4.0, "_scorer/specialty_neuro_score": 0.2622, "_scorer/specialty_neuro_std": 0.1385, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.3468, "_scorer/specialty_neurosurg_std": 0.1589, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.3852, "_scorer/specialty_obgyn_std": 0.1378, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 3.0, "_scorer/specialty_ortho_score": 0.4286, "_scorer/specialty_ortho_std": 0.1936, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.0, "_scorer/specialty_other_std": 0.0, "_scorer/specialty_other_n_samples": 9.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 1.0, "_scorer/specialty_peds_score": 0.1111, "_scorer/specialty_peds_std": 0.0954, "_scorer/specialty_peds_n_samples": 10.0, "_scorer/specialty_primary_score": 0.4941, "_scorer/specialty_primary_std": 0.1993, "_scorer/specialty_primary_n_samples": 5.0, "_scorer/specialty_psychiatry_score": 0.1111, "_scorer/specialty_psychiatry_std": 0.1028, "_scorer/specialty_psychiatry_n_samples": 9.0, "_scorer/specialty_pulm_score": 0.2, "_scorer/specialty_pulm_std": 0.1756, "_scorer/specialty_pulm_n_samples": 5.0, "_scorer/specialty_rads_score": 0.2, "_scorer/specialty_rads_std": 0.1827, "_scorer/specialty_rads_n_samples": 5.0, "_scorer/specialty_rheum_score": 0.25, "_scorer/specialty_rheum_std": 0.2151, "_scorer/specialty_rheum_n_samples": 4.0, "_scorer/specialty_surgery_score": 0.0, "_scorer/specialty_surgery_std": 0.0, "_scorer/specialty_surgery_n_samples": 7.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 3.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2529, "_scorer/source_slice_red_teaming_difficult_std": 0.0302, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.0145, "healthbench_professional_score_length_adjusted/mean": -0.068 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-24T23-44-07-00-00_healthbench-professional-physician-baseline_cY3Z8sUNPRQTpSA3UHa77J.eval", "task": "healthbench_professional_physician_baseline", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 525, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 210, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-24T23:44:07+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 4200, "total_samples": 4200, "metrics": { "_scorer/bootstrap_score": 0.4348, "_scorer/std": 0.023, "_scorer/criteria_met_rate": 0.5022, "_scorer/use_case_consult_score": 0.4758, "_scorer/use_case_consult_std": 0.0282, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.5942, "_scorer/use_case_research_std": 0.0317, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.4373, "_scorer/use_case_writing_std": 0.0364, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.5532, "_scorer/type_good_faith_std": 0.021, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4029, "_scorer/type_red_teaming_std": 0.0336, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5726, "_scorer/difficulty_typical_std": 0.0226, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.428, "_scorer/difficulty_difficult_std": 0.0269, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.1752, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.5427, "_scorer/specialty_gi_std": 0.094, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.5113, "_scorer/specialty_id_std": 0.0726, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6339, "_scorer/specialty_anesthesia_std": 0.0864, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.393, "_scorer/specialty_cards_std": 0.0608, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6483, "_scorer/specialty_derm_std": 0.0758, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4192, "_scorer/specialty_emergency_std": 0.1107, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5268, "_scorer/specialty_endo_std": 0.0865, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.6705, "_scorer/specialty_genetics_std": 0.2024, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.4844, "_scorer/specialty_heme/onc_std": 0.0841, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.6713, "_scorer/specialty_medicine_std": 0.1004, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4879, "_scorer/specialty_nephro_std": 0.0756, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5622, "_scorer/specialty_neuro_std": 0.0589, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.2723, "_scorer/specialty_neurosurg_std": 0.116, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.4284, "_scorer/specialty_obgyn_std": 0.0641, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3167, "_scorer/specialty_ophtho_std": 0.0965, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.309, "_scorer/specialty_ortho_std": 0.0834, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3395, "_scorer/specialty_other_std": 0.0885, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.6667, "_scorer/specialty_path_std": 0.2696, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.5363, "_scorer/specialty_peds_std": 0.084, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1554, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5277, "_scorer/specialty_psychiatry_std": 0.0702, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5848, "_scorer/specialty_pulm_std": 0.1138, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.5037, "_scorer/specialty_rads_std": 0.1545, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.6964, "_scorer/specialty_rheum_std": 0.1513, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.498, "_scorer/specialty_surgery_std": 0.1186, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6175, "_scorer/specialty_uro_std": 0.1097, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5726, "_scorer/source_slice_good_faith_typical_std": 0.0232, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.4897, "_scorer/source_slice_good_faith_difficult_std": 0.0533, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4029, "_scorer/source_slice_red_teaming_difficult_std": 0.0327, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.439, "healthbench_professional_score_length_adjusted/mean": 0.4348 }, "model_usage_models": [ "openai/gpt-5.4" ], "model_usage_total": { "openai/gpt-5.4": 17206029 }, "error": null, "solver": "physician_baseline_solver" }, { "space": "hs-prof-subsets", "file": "2026-07-25T00-59-37-00-00_healthbench-professional-consult_ffSg7GfyxjArrdwu7fSrBa.eval", "task": "healthbench_professional_consult", "task_version": 0, "task_file": "healthbench_professional.py", "model": "vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 236, "epochs": 8, "limit": null, "config": { "epochs": 8, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 94, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T00:59:37+00:00", "inspect_version": { "inspect_ai": "0.3.249" }, "revision": null, "completed_samples": 1888, "total_samples": 1888, "metrics": { "_scorer/bootstrap_score": 0.0819, "_scorer/std": 0.03, "_scorer/criteria_met_rate": 0.3351, "_scorer/use_case_consult_score": 0.2799, "_scorer/use_case_consult_std": 0.0253, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/type_good_faith_score": 0.3264, "_scorer/type_good_faith_std": 0.0331, "_scorer/type_good_faith_n_samples": 143.0, "_scorer/type_red_teaming_score": 0.2085, "_scorer/type_red_teaming_std": 0.0398, "_scorer/type_red_teaming_n_samples": 93.0, "_scorer/difficulty_typical_score": 0.4501, "_scorer/difficulty_typical_std": 0.0428, "_scorer/difficulty_typical_n_samples": 84.0, "_scorer/difficulty_difficult_score": 0.1859, "_scorer/difficulty_difficult_std": 0.0276, "_scorer/difficulty_difficult_n_samples": 152.0, "_scorer/specialty_ent_score": 0.7167, "_scorer/specialty_ent_std": 0.1381, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0714, "_scorer/specialty_gi_std": 0.0687, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.1667, "_scorer/specialty_id_std": 0.1545, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.2292, "_scorer/specialty_anesthesia_std": 0.1276, "_scorer/specialty_anesthesia_n_samples": 10.0, "_scorer/specialty_cards_score": 0.304, "_scorer/specialty_cards_std": 0.0899, "_scorer/specialty_cards_n_samples": 23.0, "_scorer/specialty_derm_score": 0.3059, "_scorer/specialty_derm_std": 0.0956, "_scorer/specialty_derm_n_samples": 14.0, "_scorer/specialty_emergency_score": 0.6213, "_scorer/specialty_emergency_std": 0.1507, "_scorer/specialty_emergency_n_samples": 8.0, "_scorer/specialty_endo_score": 0.4684, "_scorer/specialty_endo_std": 0.0986, "_scorer/specialty_endo_n_samples": 9.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.1, "_scorer/specialty_heme/onc_std": 0.0927, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.2537, "_scorer/specialty_medicine_std": 0.1017, "_scorer/specialty_medicine_n_samples": 6.0, "_scorer/specialty_nephro_score": 0.5461, "_scorer/specialty_nephro_std": 0.1395, "_scorer/specialty_nephro_n_samples": 9.0, "_scorer/specialty_neuro_score": 0.3117, "_scorer/specialty_neuro_std": 0.0877, "_scorer/specialty_neuro_n_samples": 15.0, "_scorer/specialty_neurosurg_score": 0.5388, "_scorer/specialty_neurosurg_std": 0.151, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.1789, "_scorer/specialty_obgyn_std": 0.0744, "_scorer/specialty_obgyn_n_samples": 18.0, "_scorer/specialty_ophtho_score": 0.2222, "_scorer/specialty_ophtho_std": 0.1368, "_scorer/specialty_ophtho_n_samples": 9.0, "_scorer/specialty_ortho_score": 0.3987, "_scorer/specialty_ortho_std": 0.1179, "_scorer/specialty_ortho_n_samples": 13.0, "_scorer/specialty_other_score": 0.1667, "_scorer/specialty_other_std": 0.1486, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.0714, "_scorer/specialty_peds_std": 0.0686, "_scorer/specialty_peds_n_samples": 14.0, "_scorer/specialty_primary_score": 0.5376, "_scorer/specialty_primary_std": 0.1833, "_scorer/specialty_primary_n_samples": 4.0, "_scorer/specialty_psychiatry_score": 0.2815, "_scorer/specialty_psychiatry_std": 0.1386, "_scorer/specialty_psychiatry_n_samples": 7.0, "_scorer/specialty_pulm_score": 0.0833, "_scorer/specialty_pulm_std": 0.0729, "_scorer/specialty_pulm_n_samples": 6.0, "_scorer/specialty_rads_score": 0.0714, "_scorer/specialty_rads_std": 0.0682, "_scorer/specialty_rads_n_samples": 7.0, "_scorer/specialty_rheum_score": 0.3524, "_scorer/specialty_rheum_std": 0.1962, "_scorer/specialty_rheum_n_samples": 5.0, "_scorer/specialty_surgery_score": 0.35, "_scorer/specialty_surgery_std": 0.1454, "_scorer/specialty_surgery_n_samples": 10.0, "_scorer/specialty_uro_score": 0.2026, "_scorer/specialty_uro_std": 0.1037, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_good_faith_typical_score": 0.4501, "_scorer/source_slice_good_faith_typical_std": 0.0412, "_scorer/source_slice_good_faith_typical_n_samples": 84.0, "_scorer/source_slice_good_faith_difficult_score": 0.1503, "_scorer/source_slice_good_faith_difficult_std": 0.0415, "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2085, "_scorer/source_slice_red_teaming_difficult_std": 0.0394, "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, "healthbench_professional_score/mean": 0.152, "healthbench_professional_score_length_adjusted/mean": 0.0819 }, "model_usage_models": [ "vllm/google/medgemma-4b-it", "openai/gpt-5.4" ], "model_usage_total": { "vllm/google/medgemma-4b-it": 21234, "openai/gpt-5.4": 54391 }, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-25T18-26-20-00-00_healthbench-professional-consult_3DSH4qRFzYfb27rH9Z9RqP.eval", "task": "healthbench_professional_consult", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 236, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": 0.1, "continue_on_fail": false, "retry_on_error": 5, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T18:26:20+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 236, "total_samples": 236, "metrics": { "_scorer/bootstrap_score": 0.2559, "_scorer/std": 0.0389, "_scorer/criteria_met_rate": 0.5483, "_scorer/use_case_consult_score": 0.4169, "_scorer/use_case_consult_std": 0.0278, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/type_good_faith_score": 0.531, "_scorer/type_good_faith_std": 0.0366, "_scorer/type_good_faith_n_samples": 143.0, "_scorer/type_red_teaming_score": 0.2414, "_scorer/type_red_teaming_std": 0.0413, "_scorer/type_red_teaming_n_samples": 93.0, "_scorer/difficulty_typical_score": 0.6563, "_scorer/difficulty_typical_std": 0.0403, "_scorer/difficulty_typical_n_samples": 84.0, "_scorer/difficulty_difficult_score": 0.2846, "_scorer/difficulty_difficult_std": 0.0343, "_scorer/difficulty_difficult_n_samples": 152.0, "_scorer/specialty_ent_score": 0.9437, "_scorer/specialty_ent_std": 0.0499, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.1429, "_scorer/specialty_gi_std": 0.1326, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.53, "_scorer/specialty_id_std": 0.1675, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.6928, "_scorer/specialty_anesthesia_std": 0.1297, "_scorer/specialty_anesthesia_n_samples": 10.0, "_scorer/specialty_cards_score": 0.267, "_scorer/specialty_cards_std": 0.0888, "_scorer/specialty_cards_n_samples": 23.0, "_scorer/specialty_derm_score": 0.5866, "_scorer/specialty_derm_std": 0.103, "_scorer/specialty_derm_n_samples": 14.0, "_scorer/specialty_emergency_score": 0.5224, "_scorer/specialty_emergency_std": 0.1573, "_scorer/specialty_emergency_n_samples": 8.0, "_scorer/specialty_endo_score": 0.5235, "_scorer/specialty_endo_std": 0.1395, "_scorer/specialty_endo_n_samples": 9.0, "_scorer/specialty_genetics_score": 0.5, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.05, "_scorer/specialty_heme/onc_std": 0.0483, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.0, "_scorer/specialty_medicine_std": 0.0, "_scorer/specialty_medicine_n_samples": 6.0, "_scorer/specialty_nephro_score": 0.3979, "_scorer/specialty_nephro_std": 0.1257, "_scorer/specialty_nephro_n_samples": 9.0, "_scorer/specialty_neuro_score": 0.5716, "_scorer/specialty_neuro_std": 0.0996, "_scorer/specialty_neuro_n_samples": 15.0, "_scorer/specialty_neurosurg_score": 0.7317, "_scorer/specialty_neurosurg_std": 0.1143, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.4717, "_scorer/specialty_obgyn_std": 0.0934, "_scorer/specialty_obgyn_n_samples": 18.0, "_scorer/specialty_ophtho_score": 0.2622, "_scorer/specialty_ophtho_std": 0.1367, "_scorer/specialty_ophtho_n_samples": 9.0, "_scorer/specialty_ortho_score": 0.5031, "_scorer/specialty_ortho_std": 0.1293, "_scorer/specialty_ortho_n_samples": 13.0, "_scorer/specialty_other_score": 0.3333, "_scorer/specialty_other_std": 0.1959, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.2222, "_scorer/specialty_peds_std": 0.1072, "_scorer/specialty_peds_n_samples": 14.0, "_scorer/specialty_primary_score": 0.5376, "_scorer/specialty_primary_std": 0.1921, "_scorer/specialty_primary_n_samples": 4.0, "_scorer/specialty_psychiatry_score": 0.572, "_scorer/specialty_psychiatry_std": 0.1603, "_scorer/specialty_psychiatry_n_samples": 7.0, "_scorer/specialty_pulm_score": 0.3229, "_scorer/specialty_pulm_std": 0.1496, "_scorer/specialty_pulm_n_samples": 6.0, "_scorer/specialty_rads_score": 0.3571, "_scorer/specialty_rads_std": 0.1658, "_scorer/specialty_rads_n_samples": 7.0, "_scorer/specialty_rheum_score": 0.5238, "_scorer/specialty_rheum_std": 0.2013, "_scorer/specialty_rheum_n_samples": 5.0, "_scorer/specialty_surgery_score": 0.5, "_scorer/specialty_surgery_std": 0.1601, "_scorer/specialty_surgery_n_samples": 10.0, "_scorer/specialty_uro_score": 0.5, "_scorer/specialty_uro_std": 0.2425, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_good_faith_typical_score": 0.6563, "_scorer/source_slice_good_faith_typical_std": 0.0386, "_scorer/source_slice_good_faith_typical_n_samples": 84.0, "_scorer/source_slice_good_faith_difficult_score": 0.3526, "_scorer/source_slice_good_faith_difficult_std": 0.0587, "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2414, "_scorer/source_slice_red_teaming_difficult_std": 0.0408, "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, "healthbench_professional_score/mean": 0.3123, "healthbench_professional_score_length_adjusted/mean": 0.2559 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-25T18-26-25-00-00_healthbench-professional-writing_2LJ8vJpJXUX2TbtokQzbxp.eval", "task": "healthbench_professional_writing", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 142, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": 0.1, "continue_on_fail": false, "retry_on_error": 5, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T18:26:25+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 142, "total_samples": 142, "metrics": { "_scorer/bootstrap_score": 0.0516, "_scorer/std": 0.0493, "_scorer/criteria_met_rate": 0.5729, "_scorer/use_case_writing_score": 0.3413, "_scorer/use_case_writing_std": 0.0356, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_red_teaming_score": 0.1317, "_scorer/type_red_teaming_std": 0.0337, "_scorer/type_red_teaming_n_samples": 91.0, "_scorer/type_good_faith_score": 0.7153, "_scorer/type_good_faith_std": 0.0482, "_scorer/type_good_faith_n_samples": 51.0, "_scorer/difficulty_difficult_score": 0.1457, "_scorer/difficulty_difficult_std": 0.033, "_scorer/difficulty_difficult_n_samples": 96.0, "_scorer/difficulty_typical_score": 0.7495, "_scorer/difficulty_typical_std": 0.0448, "_scorer/difficulty_typical_n_samples": 46.0, "_scorer/specialty_ent_score": 0.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 2.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 5.0, "_scorer/specialty_id_score": 0.1875, "_scorer/specialty_id_std": 0.1245, "_scorer/specialty_id_n_samples": 8.0, "_scorer/specialty_anesthesia_score": 0.2545, "_scorer/specialty_anesthesia_std": 0.1292, "_scorer/specialty_anesthesia_n_samples": 4.0, "_scorer/specialty_cards_score": 0.625, "_scorer/specialty_cards_std": 0.1821, "_scorer/specialty_cards_n_samples": 6.0, "_scorer/specialty_derm_score": 0.1429, "_scorer/specialty_derm_std": 0.1288, "_scorer/specialty_derm_n_samples": 7.0, "_scorer/specialty_emergency_score": 0.1429, "_scorer/specialty_emergency_std": 0.1336, "_scorer/specialty_emergency_n_samples": 7.0, "_scorer/specialty_endo_score": 0.3474, "_scorer/specialty_endo_std": 0.1927, "_scorer/specialty_endo_n_samples": 5.0, "_scorer/specialty_genetics_score": 0.0, "_scorer/specialty_genetics_std": 0.0, "_scorer/specialty_genetics_n_samples": 1.0, "_scorer/specialty_heme/onc_score": 0.4362, "_scorer/specialty_heme/onc_std": 0.1471, "_scorer/specialty_heme/onc_n_samples": 10.0, "_scorer/specialty_medicine_score": 0.3333, "_scorer/specialty_medicine_std": 0.2676, "_scorer/specialty_medicine_n_samples": 3.0, "_scorer/specialty_nephro_score": 0.3914, "_scorer/specialty_nephro_std": 0.1664, "_scorer/specialty_nephro_n_samples": 6.0, "_scorer/specialty_neuro_score": 0.414, "_scorer/specialty_neuro_std": 0.1342, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.25, "_scorer/specialty_neurosurg_std": 0.2135, "_scorer/specialty_neurosurg_n_samples": 4.0, "_scorer/specialty_obgyn_score": 0.5399, "_scorer/specialty_obgyn_std": 0.1231, "_scorer/specialty_obgyn_n_samples": 12.0, "_scorer/specialty_ophtho_score": 0.5417, "_scorer/specialty_ophtho_std": 0.2039, "_scorer/specialty_ophtho_n_samples": 5.0, "_scorer/specialty_ortho_score": 0.25, "_scorer/specialty_ortho_std": 0.2196, "_scorer/specialty_ortho_n_samples": 4.0, "_scorer/specialty_other_score": 0.2917, "_scorer/specialty_other_std": 0.1496, "_scorer/specialty_other_n_samples": 8.0, "_scorer/specialty_peds_score": 0.6875, "_scorer/specialty_peds_std": 0.1536, "_scorer/specialty_peds_n_samples": 8.0, "_scorer/specialty_primary_score": 0.3333, "_scorer/specialty_primary_std": 0.2706, "_scorer/specialty_primary_n_samples": 3.0, "_scorer/specialty_psychiatry_score": 0.4611, "_scorer/specialty_psychiatry_std": 0.1256, "_scorer/specialty_psychiatry_n_samples": 12.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 2.0, "_scorer/specialty_rads_score": 0.4667, "_scorer/specialty_rads_std": 0.0, "_scorer/specialty_rads_n_samples": 1.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 2.0, "_scorer/specialty_surgery_score": 0.25, "_scorer/specialty_surgery_std": 0.2181, "_scorer/specialty_surgery_n_samples": 4.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 4.0, "_scorer/source_slice_red_teaming_difficult_score": 0.1317, "_scorer/source_slice_red_teaming_difficult_std": 0.034, "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, "_scorer/source_slice_good_faith_difficult_score": 0.4, "_scorer/source_slice_good_faith_difficult_std": 0.2235, "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, "_scorer/source_slice_good_faith_typical_score": 0.7495, "_scorer/source_slice_good_faith_typical_std": 0.045, "_scorer/source_slice_good_faith_typical_n_samples": 46.0, "healthbench_professional_score/mean": 0.0973, "healthbench_professional_score_length_adjusted/mean": 0.0516 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-25T18-26-28-00-00_healthbench-professional-research_9Pb5iwuDu8dmuFzs5FnmBw.eval", "task": "healthbench_professional_research", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 147, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": 0.1, "continue_on_fail": false, "retry_on_error": 5, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T18:26:28+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 147, "total_samples": 147, "metrics": { "_scorer/bootstrap_score": 0.5185, "_scorer/std": 0.0346, "_scorer/criteria_met_rate": 0.6757, "_scorer/use_case_research_score": 0.6383, "_scorer/use_case_research_std": 0.0316, "_scorer/use_case_research_n_samples": 147.0, "_scorer/type_red_teaming_score": 0.8571, "_scorer/type_red_teaming_std": 0.1322, "_scorer/type_red_teaming_n_samples": 7.0, "_scorer/type_good_faith_score": 0.6274, "_scorer/type_good_faith_std": 0.034, "_scorer/type_good_faith_n_samples": 140.0, "_scorer/difficulty_difficult_score": 0.5476, "_scorer/difficulty_difficult_std": 0.1009, "_scorer/difficulty_difficult_n_samples": 21.0, "_scorer/difficulty_typical_score": 0.6534, "_scorer/difficulty_typical_std": 0.0323, "_scorer/difficulty_typical_n_samples": 126.0, "_scorer/specialty_ent_score": 1.0, "_scorer/specialty_ent_std": 0.0, "_scorer/specialty_ent_n_samples": 1.0, "_scorer/specialty_gi_score": 0.4254, "_scorer/specialty_gi_std": 0.1646, "_scorer/specialty_gi_n_samples": 7.0, "_scorer/specialty_id_score": 0.5635, "_scorer/specialty_id_std": 0.1747, "_scorer/specialty_id_n_samples": 6.0, "_scorer/specialty_anesthesia_score": 0.6875, "_scorer/specialty_anesthesia_std": 0.1483, "_scorer/specialty_anesthesia_n_samples": 8.0, "_scorer/specialty_cards_score": 0.6477, "_scorer/specialty_cards_std": 0.0877, "_scorer/specialty_cards_n_samples": 14.0, "_scorer/specialty_derm_score": 0.75, "_scorer/specialty_derm_std": 0.2211, "_scorer/specialty_derm_n_samples": 4.0, "_scorer/specialty_emergency_score": 0.0, "_scorer/specialty_emergency_std": 0.0, "_scorer/specialty_emergency_n_samples": 1.0, "_scorer/specialty_endo_score": 0.7733, "_scorer/specialty_endo_std": 0.0922, "_scorer/specialty_endo_n_samples": 6.0, "_scorer/specialty_genetics_score": 0.8409, "_scorer/specialty_genetics_std": 0.1115, "_scorer/specialty_genetics_n_samples": 2.0, "_scorer/specialty_heme/onc_score": 0.651, "_scorer/specialty_heme/onc_std": 0.1332, "_scorer/specialty_heme/onc_n_samples": 7.0, "_scorer/specialty_medicine_score": 0.427, "_scorer/specialty_medicine_std": 0.1528, "_scorer/specialty_medicine_n_samples": 8.0, "_scorer/specialty_nephro_score": 0.8538, "_scorer/specialty_nephro_std": 0.0877, "_scorer/specialty_nephro_n_samples": 7.0, "_scorer/specialty_neuro_score": 0.7029, "_scorer/specialty_neuro_std": 0.0808, "_scorer/specialty_neuro_n_samples": 19.0, "_scorer/specialty_neurosurg_score": 0.0667, "_scorer/specialty_neurosurg_std": 0.0, "_scorer/specialty_neurosurg_n_samples": 1.0, "_scorer/specialty_obgyn_score": 0.5013, "_scorer/specialty_obgyn_std": 0.1234, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 1.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 4.0, "_scorer/specialty_ortho_score": 0.3345, "_scorer/specialty_ortho_std": 0.1254, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.6267, "_scorer/specialty_other_std": 0.1434, "_scorer/specialty_other_n_samples": 6.0, "_scorer/specialty_peds_score": 0.7143, "_scorer/specialty_peds_std": 0.1761, "_scorer/specialty_peds_n_samples": 6.0, "_scorer/specialty_psychiatry_score": 0.6783, "_scorer/specialty_psychiatry_std": 0.1318, "_scorer/specialty_psychiatry_n_samples": 10.0, "_scorer/specialty_pulm_score": 0.5474, "_scorer/specialty_pulm_std": 0.1845, "_scorer/specialty_pulm_n_samples": 4.0, "_scorer/specialty_rheum_score": 1.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 1.0, "_scorer/specialty_surgery_score": 1.0, "_scorer/specialty_surgery_std": 0.0, "_scorer/specialty_surgery_n_samples": 3.0, "_scorer/specialty_uro_score": 0.5675, "_scorer/specialty_uro_std": 0.1738, "_scorer/specialty_uro_n_samples": 5.0, "_scorer/source_slice_red_teaming_difficult_score": 0.8571, "_scorer/source_slice_red_teaming_difficult_std": 0.1341, "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, "_scorer/source_slice_good_faith_typical_score": 0.6534, "_scorer/source_slice_good_faith_typical_std": 0.0325, "_scorer/source_slice_good_faith_typical_n_samples": 126.0, "_scorer/source_slice_good_faith_difficult_score": 0.3929, "_scorer/source_slice_good_faith_difficult_std": 0.1225, "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, "healthbench_professional_score/mean": 0.6298, "healthbench_professional_score_length_adjusted/mean": 0.5185 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-25T18-26-32-00-00_healthbench-professional-red-teaming_e9hCTjjRXrNTnexAxrYM8x.eval", "task": "healthbench_professional_red_teaming", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 191, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": 0.1, "continue_on_fail": false, "retry_on_error": 5, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T18:26:32+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 191, "total_samples": 191, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0067, "_scorer/criteria_met_rate": 0.4643, "_scorer/use_case_consult_score": 0.2414, "_scorer/use_case_consult_std": 0.0408, "_scorer/use_case_consult_n_samples": 93.0, "_scorer/use_case_research_score": 0.8571, "_scorer/use_case_research_std": 0.1336, "_scorer/use_case_research_n_samples": 7.0, "_scorer/use_case_writing_score": 0.1317, "_scorer/use_case_writing_std": 0.0327, "_scorer/use_case_writing_n_samples": 91.0, "_scorer/type_red_teaming_score": 0.2117, "_scorer/type_red_teaming_std": 0.0287, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_difficult_score": 0.2117, "_scorer/difficulty_difficult_std": 0.0281, "_scorer/difficulty_difficult_n_samples": 191.0, "_scorer/specialty_ent_score": 0.5, "_scorer/specialty_ent_std": 0.2521, "_scorer/specialty_ent_n_samples": 4.0, "_scorer/specialty_gi_score": 0.0, "_scorer/specialty_gi_std": 0.0, "_scorer/specialty_gi_n_samples": 8.0, "_scorer/specialty_id_score": 0.25, "_scorer/specialty_id_std": 0.1261, "_scorer/specialty_id_n_samples": 10.0, "_scorer/specialty_allergy_score": 0.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 1.0, "_scorer/specialty_anesthesia_score": 0.5238, "_scorer/specialty_anesthesia_std": 0.1756, "_scorer/specialty_anesthesia_n_samples": 7.0, "_scorer/specialty_cards_score": 0.0909, "_scorer/specialty_cards_std": 0.0927, "_scorer/specialty_cards_n_samples": 11.0, "_scorer/specialty_derm_score": 0.1845, "_scorer/specialty_derm_std": 0.0962, "_scorer/specialty_derm_n_samples": 11.0, "_scorer/specialty_emergency_score": 0.1818, "_scorer/specialty_emergency_std": 0.1166, "_scorer/specialty_emergency_n_samples": 11.0, "_scorer/specialty_endo_score": 0.3429, "_scorer/specialty_endo_std": 0.1484, "_scorer/specialty_endo_n_samples": 10.0, "_scorer/specialty_genetics_score": 0.5, "_scorer/specialty_genetics_std": 0.2312, "_scorer/specialty_genetics_n_samples": 3.0, "_scorer/specialty_heme/onc_score": 0.2273, "_scorer/specialty_heme/onc_std": 0.1173, "_scorer/specialty_heme/onc_n_samples": 11.0, "_scorer/specialty_medicine_score": 0.1429, "_scorer/specialty_medicine_std": 0.1361, "_scorer/specialty_medicine_n_samples": 7.0, "_scorer/specialty_nephro_score": 0.0, "_scorer/specialty_nephro_std": 0.0, "_scorer/specialty_nephro_n_samples": 4.0, "_scorer/specialty_neuro_score": 0.1511, "_scorer/specialty_neuro_std": 0.109, "_scorer/specialty_neuro_n_samples": 9.0, "_scorer/specialty_neurosurg_score": 0.4762, "_scorer/specialty_neurosurg_std": 0.1635, "_scorer/specialty_neurosurg_n_samples": 6.0, "_scorer/specialty_obgyn_score": 0.4018, "_scorer/specialty_obgyn_std": 0.1363, "_scorer/specialty_obgyn_n_samples": 10.0, "_scorer/specialty_ophtho_score": 0.0, "_scorer/specialty_ophtho_std": 0.0, "_scorer/specialty_ophtho_n_samples": 3.0, "_scorer/specialty_ortho_score": 0.2857, "_scorer/specialty_ortho_std": 0.1683, "_scorer/specialty_ortho_n_samples": 7.0, "_scorer/specialty_other_score": 0.0, "_scorer/specialty_other_std": 0.0, "_scorer/specialty_other_n_samples": 9.0, "_scorer/specialty_path_score": 0.0, "_scorer/specialty_path_std": 0.0, "_scorer/specialty_path_n_samples": 1.0, "_scorer/specialty_peds_score": 0.2111, "_scorer/specialty_peds_std": 0.1252, "_scorer/specialty_peds_n_samples": 10.0, "_scorer/specialty_primary_score": 0.4, "_scorer/specialty_primary_std": 0.2224, "_scorer/specialty_primary_n_samples": 5.0, "_scorer/specialty_psychiatry_score": 0.2222, "_scorer/specialty_psychiatry_std": 0.1382, "_scorer/specialty_psychiatry_n_samples": 9.0, "_scorer/specialty_pulm_score": 0.0, "_scorer/specialty_pulm_std": 0.0, "_scorer/specialty_pulm_n_samples": 5.0, "_scorer/specialty_rads_score": 0.2933, "_scorer/specialty_rads_std": 0.1814, "_scorer/specialty_rads_n_samples": 5.0, "_scorer/specialty_rheum_score": 0.0, "_scorer/specialty_rheum_std": 0.0, "_scorer/specialty_rheum_n_samples": 4.0, "_scorer/specialty_surgery_score": 0.1429, "_scorer/specialty_surgery_std": 0.1288, "_scorer/specialty_surgery_n_samples": 7.0, "_scorer/specialty_uro_score": 0.0, "_scorer/specialty_uro_std": 0.0, "_scorer/specialty_uro_n_samples": 3.0, "_scorer/source_slice_red_teaming_difficult_score": 0.2117, "_scorer/source_slice_red_teaming_difficult_std": 0.028, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": -0.0532, "healthbench_professional_score_length_adjusted/mean": -0.0829 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "generate" }, { "space": "hs-prof-subsets", "file": "2026-07-25T18-26-35-00-00_healthbench-professional-physician-baseline_Njv8PLSvCJpAvUbPPMQqkx.eval", "task": "healthbench_professional_physician_baseline", "task_version": 0, "task_file": "healthbench_professional.py", "model": "openrouter/deepseek/deepseek-v4-pro", "model_args": {}, "model_generate_config": {}, "task_args": { "kwargs": { "judge_model": "openai/gpt-5.4", "judge_reasoning_effort": "low" } }, "dataset_samples": 525, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": 0.1, "continue_on_fail": false, "retry_on_error": 5, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-25T18:26:35+00:00", "inspect_version": { "inspect_ai": "0.3.246" }, "revision": null, "completed_samples": 525, "total_samples": 525, "metrics": { "_scorer/bootstrap_score": 0.4289, "_scorer/std": 0.0236, "_scorer/criteria_met_rate": 0.5057, "_scorer/use_case_consult_score": 0.4776, "_scorer/use_case_consult_std": 0.0275, "_scorer/use_case_consult_n_samples": 236.0, "_scorer/use_case_research_score": 0.5781, "_scorer/use_case_research_std": 0.0331, "_scorer/use_case_research_n_samples": 147.0, "_scorer/use_case_writing_score": 0.4343, "_scorer/use_case_writing_std": 0.0345, "_scorer/use_case_writing_n_samples": 142.0, "_scorer/type_good_faith_score": 0.547, "_scorer/type_good_faith_std": 0.0211, "_scorer/type_good_faith_n_samples": 334.0, "_scorer/type_red_teaming_score": 0.4013, "_scorer/type_red_teaming_std": 0.0319, "_scorer/type_red_teaming_n_samples": 191.0, "_scorer/difficulty_typical_score": 0.5684, "_scorer/difficulty_typical_std": 0.0232, "_scorer/difficulty_typical_n_samples": 256.0, "_scorer/difficulty_difficult_score": 0.4232, "_scorer/difficulty_difficult_std": 0.0281, "_scorer/difficulty_difficult_n_samples": 269.0, "_scorer/specialty_ent_score": 0.4857, "_scorer/specialty_ent_std": 0.1722, "_scorer/specialty_ent_n_samples": 7.0, "_scorer/specialty_gi_score": 0.5953, "_scorer/specialty_gi_std": 0.0941, "_scorer/specialty_gi_n_samples": 19.0, "_scorer/specialty_id_score": 0.5394, "_scorer/specialty_id_std": 0.0696, "_scorer/specialty_id_n_samples": 20.0, "_scorer/specialty_allergy_score": 1.0, "_scorer/specialty_allergy_std": 0.0, "_scorer/specialty_allergy_n_samples": 2.0, "_scorer/specialty_anesthesia_score": 0.5969, "_scorer/specialty_anesthesia_std": 0.0908, "_scorer/specialty_anesthesia_n_samples": 22.0, "_scorer/specialty_cards_score": 0.393, "_scorer/specialty_cards_std": 0.0609, "_scorer/specialty_cards_n_samples": 43.0, "_scorer/specialty_derm_score": 0.6083, "_scorer/specialty_derm_std": 0.0803, "_scorer/specialty_derm_n_samples": 25.0, "_scorer/specialty_emergency_score": 0.4612, "_scorer/specialty_emergency_std": 0.1168, "_scorer/specialty_emergency_n_samples": 16.0, "_scorer/specialty_endo_score": 0.5108, "_scorer/specialty_endo_std": 0.0824, "_scorer/specialty_endo_n_samples": 20.0, "_scorer/specialty_genetics_score": 0.6705, "_scorer/specialty_genetics_std": 0.2035, "_scorer/specialty_genetics_n_samples": 4.0, "_scorer/specialty_heme/onc_score": 0.4659, "_scorer/specialty_heme/onc_std": 0.0861, "_scorer/specialty_heme/onc_n_samples": 27.0, "_scorer/specialty_medicine_score": 0.6713, "_scorer/specialty_medicine_std": 0.0984, "_scorer/specialty_medicine_n_samples": 17.0, "_scorer/specialty_nephro_score": 0.4311, "_scorer/specialty_nephro_std": 0.0782, "_scorer/specialty_nephro_n_samples": 22.0, "_scorer/specialty_neuro_score": 0.5338, "_scorer/specialty_neuro_std": 0.0636, "_scorer/specialty_neuro_n_samples": 43.0, "_scorer/specialty_neurosurg_score": 0.3242, "_scorer/specialty_neurosurg_std": 0.116, "_scorer/specialty_neurosurg_n_samples": 11.0, "_scorer/specialty_obgyn_score": 0.42, "_scorer/specialty_obgyn_std": 0.0637, "_scorer/specialty_obgyn_n_samples": 40.0, "_scorer/specialty_ophtho_score": 0.3167, "_scorer/specialty_ophtho_std": 0.0949, "_scorer/specialty_ophtho_n_samples": 18.0, "_scorer/specialty_ortho_score": 0.309, "_scorer/specialty_ortho_std": 0.081, "_scorer/specialty_ortho_n_samples": 24.0, "_scorer/specialty_other_score": 0.3541, "_scorer/specialty_other_std": 0.0886, "_scorer/specialty_other_n_samples": 20.0, "_scorer/specialty_path_score": 0.6667, "_scorer/specialty_path_std": 0.2659, "_scorer/specialty_path_n_samples": 3.0, "_scorer/specialty_peds_score": 0.5542, "_scorer/specialty_peds_std": 0.0832, "_scorer/specialty_peds_n_samples": 28.0, "_scorer/specialty_primary_score": 0.563, "_scorer/specialty_primary_std": 0.1594, "_scorer/specialty_primary_n_samples": 7.0, "_scorer/specialty_psychiatry_score": 0.5439, "_scorer/specialty_psychiatry_std": 0.072, "_scorer/specialty_psychiatry_n_samples": 29.0, "_scorer/specialty_pulm_score": 0.5848, "_scorer/specialty_pulm_std": 0.1168, "_scorer/specialty_pulm_n_samples": 12.0, "_scorer/specialty_rads_score": 0.3787, "_scorer/specialty_rads_std": 0.1462, "_scorer/specialty_rads_n_samples": 8.0, "_scorer/specialty_rheum_score": 0.6964, "_scorer/specialty_rheum_std": 0.1571, "_scorer/specialty_rheum_n_samples": 8.0, "_scorer/specialty_surgery_score": 0.498, "_scorer/specialty_surgery_std": 0.1167, "_scorer/specialty_surgery_n_samples": 17.0, "_scorer/specialty_uro_score": 0.6175, "_scorer/specialty_uro_std": 0.1148, "_scorer/specialty_uro_n_samples": 13.0, "_scorer/source_slice_good_faith_typical_score": 0.5684, "_scorer/source_slice_good_faith_typical_std": 0.0225, "_scorer/source_slice_good_faith_typical_n_samples": 256.0, "_scorer/source_slice_good_faith_difficult_score": 0.4768, "_scorer/source_slice_good_faith_difficult_std": 0.0535, "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, "_scorer/source_slice_red_teaming_difficult_score": 0.4013, "_scorer/source_slice_red_teaming_difficult_std": 0.0314, "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, "healthbench_professional_score/mean": 0.4331, "healthbench_professional_score_length_adjusted/mean": 0.4289 }, "model_usage_models": [], "model_usage_total": {}, "error": null, "solver": "physician_baseline_solver" }, { "space": "medgemma-healthbench-hard", "file": "2026-08-05T06-05-04-00-00_healthbench-hard_U2qKVPncnWPTbmLEfZfa4y.eval", "task": "healthbench_hard", "task_version": 2, "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", "model": "openai-api/vllm/google/medgemma-27b-text-it", "model_args": {}, "model_generate_config": { "max_connections": 16, "max_tokens": 4096, "temperature": 0.0 }, "task_args": { "judge_model": "openai/gpt-4.1", "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "retry_on_error": 3, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-05T06:05:04+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.1332, "_scorer/std": 0.0102, "_scorer/criteria_met_rate": 0.4424, "healthbench_score/mean": 0.1332, "healthbench_score_length_adjusted/mean": -0.0442 }, "model_usage_models": [ "openai-api/vllm/google/medgemma-27b-text-it", "openai/gpt-4.1" ], "model_usage_total": { "openai-api/vllm/google/medgemma-27b-text-it": 1108556, "openai/gpt-4.1": 22501745 }, "error": null, "solver": "generate" }, { "space": "medgemma-healthbench-hard", "file": "2026-08-05T07-09-21-00-00_healthbench-hard_4bZEuQss4Bbee8KixwTS4o.eval", "task": "healthbench_hard", "task_version": 2, "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", "model": "openai-api/vllm/google/medgemma-4b-it", "model_args": {}, "model_generate_config": { "max_connections": 16, "max_tokens": 4096, "temperature": 0.0 }, "task_args": { "judge_model": "openai/gpt-4.1", "length_adjustment_center": 2000, "length_adjustment_penalty_per_500_chars": 0.0392 }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "retry_on_error": 3, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-08-05T07:09:21+00:00", "inspect_version": { "inspect_ai": "0.3.252" }, "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", "completed_samples": 1000, "total_samples": 1000, "metrics": { "_scorer/bootstrap_score": 0.0, "_scorer/std": 0.0, "_scorer/criteria_met_rate": 0.358, "healthbench_score/mean": -0.0312, "healthbench_score_length_adjusted/mean": -0.1641 }, "model_usage_models": [ "openai-api/vllm/google/medgemma-4b-it", "openai/gpt-4.1" ], "model_usage_total": { "openai-api/vllm/google/medgemma-4b-it": 918963, "openai/gpt-4.1": 20096686 }, "error": null, "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T08-07-45-00-00_healthbench-hard_VLbsW5zXdzSbfgeNMziFZ2.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "openai/gpt-5-nano", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "error", "created": "2026-07-09T08:07:45+00:00", "inspect_version": { "inspect_ai": "0.3.244", "inspect_evals": "0.14.2" }, "revision": "https://github.com/JonathanGabor/evilgenie_inspect.git", "completed_samples": null, "total_samples": null, "metrics": {}, "model_usage_models": [], "model_usage_total": {}, "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T08-10-01-00-00_healthbench-hard_jofngniV8jDhpdvbXSg9S4.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "error", "created": "2026-07-09T08:10:01+00:00", "inspect_version": { "inspect_ai": "0.3.244", "inspect_evals": "0.14.2" }, "revision": "https://github.com/JonathanGabor/evilgenie_inspect.git", "completed_samples": null, "total_samples": null, "metrics": {}, "model_usage_models": [], "model_usage_total": {}, "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T08-11-34-00-00_healthbench-hard_CzHRcCD6CoJpUFoxxWadFn.eval", "task": "inspect_evals/healthbench_hard", "task_version": 2, "task_file": null, "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 1000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 50, "log_shared": 0, "score_display": true }, "status": "cancelled", "created": "2026-07-09T08:11:34+00:00", "inspect_version": { "inspect_ai": "0.3.244", "inspect_evals": "0.14.2" }, "revision": "https://github.com/JonathanGabor/evilgenie_inspect.git", "completed_samples": null, "total_samples": null, "metrics": {}, "model_usage_models": [ "anthropic/claude-opus-4-7" ], "model_usage_total": { "anthropic/claude-opus-4-7": 46358 }, "error": null, "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T09-03-02-00-00_healthbench_EVLdZCZJQYG5EuWtS2TLsK.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4o-mini", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "error", "created": "2026-07-09T09:03:02+00:00", "inspect_version": { "inspect_ai": "0.3.245", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": null, "total_samples": null, "metrics": {}, "model_usage_models": [], "model_usage_total": {}, "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T09-05-32-00-00_healthbench_Aucfe2K5BgdpYrkukVNJcf.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4o-mini", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "error", "created": "2026-07-09T09:05:32+00:00", "inspect_version": { "inspect_ai": "0.3.245", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": null, "total_samples": null, "metrics": {}, "model_usage_models": [], "model_usage_total": {}, "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T09-09-45-00-00_healthbench_gGjNAEJ4UezLEQvJPVi7Zi.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "openai/gpt-5.5", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4o-mini", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-09T09:09:45+00:00", "inspect_version": { "inspect_ai": "0.3.245", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.4873, "_scorer/std": 0.0045, "_scorer/criteria_met_rate": 0.4846, "healthbench_score/mean": 0.4873 }, "model_usage_models": [ "openai/gpt-5.5", "openai/gpt-4o-mini" ], "model_usage_total": { "openai/gpt-5.5": 4699836, "openai/gpt-4o-mini": 84688797 }, "error": null, "solver": "generate" }, { "space": "healthbench-v1", "file": "2026-07-09T10-33-03-00-00_healthbench_QeHrUjumcLfkEWL5T7HaUa.eval", "task": "inspect_evals/healthbench", "task_version": 2, "task_file": null, "model": "anthropic/claude-opus-4-7", "model_args": {}, "model_generate_config": {}, "task_args": { "subset": "full", "local_path": null, "judge_model": "openai/gpt-4o-mini", "judge_max_tokens": null, "judge_temperature": 0.0, "judge_system_message": "You are a helpful assistant.", "bootstrap_samples": 1000, "include_subset_scores": false, "length_adjustment_center": null, "length_adjustment_penalty_per_500_chars": null }, "dataset_samples": 5000, "epochs": 1, "limit": null, "config": { "epochs": 1, "fail_on_error": true, "continue_on_fail": false, "sandbox_cleanup": true, "log_samples": true, "log_realtime": true, "log_images": true, "log_buffer": 250, "log_shared": 0, "score_display": true }, "status": "success", "created": "2026-07-09T10:33:03+00:00", "inspect_version": { "inspect_ai": "0.3.245", "inspect_evals": "0.14.3" }, "revision": null, "completed_samples": 5000, "total_samples": 5000, "metrics": { "_scorer/bootstrap_score": 0.4762, "_scorer/std": 0.0043, "_scorer/criteria_met_rate": 0.4723, "healthbench_score/mean": 0.4762 }, "model_usage_models": [ "anthropic/claude-opus-4-7", "openai/gpt-4o-mini" ], "model_usage_total": { "anthropic/claude-opus-4-7": 4923085, "openai/gpt-4o-mini": 80274748 }, "error": null, "solver": "generate" } ]