{ "version": "research-v1.1-scoring", "execution_version": "research-v1-diagnostic", "generated_at": "2026-09-07T03:38:25.917892+00:00", "planned_runs": 120, "scoring_erratum": "SCORING-ERRATUM.md", "aggregation": "All planned target slots and candidate decisions remain in denominators; infrastructure failures are unanswered, not attributed model factual errors.", "source_commit": "9d7619787d3cdfb2a4df46e1cbb522bcb75013bf", "qualification": "All 24 cases are author-reviewed synthetic diagnostics. Independent CRE review and held-out qualification have not occurred.", "conditions": [ "Six variants per task share a generator; 24 tasks are not 24 independent real deals", "Structured source extracts supplied inline; no native file-ingestion score", "Native tool access is audited after execution; violations are shown and excluded from same-data claims", "Direct controls exclude artifact generation; native and generic agent artifacts assessed separately", "Original research design proposed independent review/held-out split; this release is explicitly diagnostic instead" ], "case_ids": [ "refinance-leads-alder", "refinance-leads-birch", "refinance-leads-cypress", "refinance-leads-dogwood", "refinance-leads-elm", "refinance-leads-fir", "rent-comps-alder", "rent-comps-birch", "rent-comps-cypress", "rent-comps-dogwood", "rent-comps-elm", "rent-comps-fir", "sales-comps-alder", "sales-comps-birch", "sales-comps-cypress", "sales-comps-dogwood", "sales-comps-elm", "sales-comps-fir", "sponsor-leads-alder", "sponsor-leads-birch", "sponsor-leads-cypress", "sponsor-leads-dogwood", "sponsor-leads-elm", "sponsor-leads-fir" ], "runs": [ { "case_id": "refinance-leads-alder", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:26:51.270Z", "observed_completed_at": "2026-09-07T02:29:45.989Z", "packet_sha256": "b06ca3a6283c85446e3988378a4b85713c6bb273b91c2cac222518f90aa53e0a", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/refinance_leads_alder_charlotte.csv", "sha256": "9581880f421768c9b764c6b4e9ef9bd22a92290368d8496669e7f01806aa8fae", "bytes": 1381 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.4377025, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 99.5, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "8cf87205-1fe8-41c4-a671-a47f6e4c5496", "session_id": "e73701f9-149e-4116-9684-3ab0b6f388a1", "started_at": "2026-09-06T22:27:57.385000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.29788725, 0.13981525 ], "total_cost_usd": 0.4377025, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 99.5, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/refinance-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-alder", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R1514", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1514" ], "reported_sources": [ "S-R1514" ] }, { "record_id": "R4598", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4598" ], "reported_sources": [ "S-R4598" ] }, { "record_id": "R7567", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 17640000, "expected": 17640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7567" ], "reported_sources": [ "S-R7567" ] }, { "record_id": "R8446", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 18720000, "expected": 18720000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8446" ], "reported_sources": [ "S-R8446" ] }, { "record_id": "R2663", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 19800000, "expected": 19800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2663" ], "reported_sources": [ "S-R2663" ] }, { "record_id": "R3927", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 20880000, "expected": 20880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3927" ], "reported_sources": [ "S-R3927" ] } ], "decision_checks": [ { "record_id": "R2193", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6607", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5582", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R9859", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4360", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6400", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R8446", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2682", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R1514", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8411", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2324", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R2663", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6902", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7567", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6774", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3086", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R4598", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1100", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/refinance_leads_alder_charlotte.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "R1514 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R4598 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7567 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8446 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8446 / contact_email: artifact differs from answer or lacks explicit unknown", "R2663 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3927 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R1514", "R4598", "R7567", "R8446", "R2663", "R3927" ], "answer_record_order": [ "R1514", "R4598", "R7567", "R8446", "R2663", "R3927" ], "field_checks": [ { "record_id": "R1514", "field": "entity_id", "actual": "ALDER-REF-61562", "original": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "record_id": "R1514", "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "original": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "record_id": "R1514", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R1514", "field": "original_principal_usd", "actual": "14400000", "original": "14400000", "expected": 14400000, "passed": true }, { "record_id": "R1514", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R1514", "field": "contact_name", "actual": "Casey Alder1", "original": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "record_id": "R1514", "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "original": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true }, { "record_id": "R4598", "field": "entity_id", "actual": "ALDER-REF-82017", "original": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "record_id": "R4598", "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "original": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "record_id": "R4598", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R4598", "field": "original_principal_usd", "actual": "16560000", "original": "16560000", "expected": 16560000, "passed": true }, { "record_id": "R4598", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R4598", "field": "contact_name", "actual": "Avery Alder3", "original": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "record_id": "R4598", "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "original": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true }, { "record_id": "R7567", "field": "entity_id", "actual": "ALDER-REF-70342", "original": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "record_id": "R7567", "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "original": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "record_id": "R7567", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R7567", "field": "original_principal_usd", "actual": "17640000", "original": "17640000", "expected": 17640000, "passed": true }, { "record_id": "R7567", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7567", "field": "contact_name", "actual": "Sam Alder4", "original": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "record_id": "R7567", "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "original": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true }, { "record_id": "R8446", "field": "entity_id", "actual": "ALDER-REF-87904", "original": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "record_id": "R8446", "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "original": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "record_id": "R8446", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R8446", "field": "original_principal_usd", "actual": "18720000", "original": "18720000", "expected": 18720000, "passed": true }, { "record_id": "R8446", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R8446", "field": "contact_name", "actual": "Casey Alder5", "original": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "record_id": "R8446", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R2663", "field": "entity_id", "actual": "ALDER-REF-13745", "original": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "record_id": "R2663", "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "original": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "record_id": "R2663", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R2663", "field": "original_principal_usd", "actual": "19800000", "original": "19800000", "expected": 19800000, "passed": true }, { "record_id": "R2663", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R2663", "field": "contact_name", "actual": "Riley Alder6", "original": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "record_id": "R2663", "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "original": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true }, { "record_id": "R3927", "field": "entity_id", "actual": "ALDER-REF-56841", "original": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "record_id": "R3927", "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "original": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "record_id": "R3927", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R3927", "field": "original_principal_usd", "actual": "20880000", "original": "20880000", "expected": 20880000, "passed": true }, { "record_id": "R3927", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R3927", "field": "contact_name", "actual": "Avery Alder7", "original": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "record_id": "R3927", "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "original": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1514", "passed": true, "expected": [ "S-R1514" ], "actual": [ "S-R1514" ] }, { "record_id": "R4598", "passed": true, "expected": [ "S-R4598" ], "actual": [ "S-R4598" ] }, { "record_id": "R7567", "passed": true, "expected": [ "S-R7567" ], "actual": [ "S-R7567" ] }, { "record_id": "R8446", "passed": true, "expected": [ "S-R8446" ], "actual": [ "S-R8446" ] }, { "record_id": "R2663", "passed": true, "expected": [ "S-R2663" ], "actual": [ "S-R2663" ] }, { "record_id": "R3927", "passed": true, "expected": [ "S-R3927" ], "actual": [ "S-R3927" ] } ], "source_sha256": "9581880f421768c9b764c6b4e9ef9bd22a92290368d8496669e7f01806aa8fae" } }, { "case_id": "refinance-leads-alder", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:30:05.510418+00:00", "completed_at": "2026-09-07T02:37:39.599777+00:00", "elapsed_seconds": 454.087, "cost_usd": 0.5312985, "known_partial_cost_usd": 0.5312985, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 8, "tool_calls": 8, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7784, "completion_tokens": 5439, "total_tokens": 13223, "cost": 0.06412, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4992, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06412, "gateway_cost": 0.06412 }, "finish_reason": "tool_calls", "elapsed_seconds": 51.166, "completed_at": "2026-09-07T02:30:56.694816+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8251, "completion_tokens": 8014, "total_tokens": 16265, "cost": 0.08181375, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7680, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7616, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08181375, "gateway_cost": 0.08181375 }, "finish_reason": "tool_calls", "elapsed_seconds": 78.494, "completed_at": "2026-09-07T02:32:15.192558+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8661, "completion_tokens": 4984, "total_tokens": 13645, "cost": 0.05145025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8192, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4544, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05145025, "gateway_cost": 0.05145025 }, "finish_reason": "tool_calls", "elapsed_seconds": 51.29, "completed_at": "2026-09-07T02:33:06.485678+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9071, "completion_tokens": 7491, "total_tokens": 16562, "cost": 0.07660075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8576, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7040, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07660075, "gateway_cost": 0.07660075 }, "finish_reason": "tool_calls", "elapsed_seconds": 60.023, "completed_at": "2026-09-07T02:34:06.512435+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9481, "completion_tokens": 6247, "total_tokens": 15728, "cost": 0.07432125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5760, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07432125, "gateway_cost": 0.07432125 }, "finish_reason": "tool_calls", "elapsed_seconds": 59.22, "completed_at": "2026-09-07T02:35:05.736836+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9948, "completion_tokens": 4822, "total_tokens": 14770, "cost": 0.050143, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9344, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4416, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.050143, "gateway_cost": 0.050143 }, "finish_reason": "tool_calls", "elapsed_seconds": 43.732, "completed_at": "2026-09-07T02:35:49.472738+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 10358, "completion_tokens": 6067, "total_tokens": 16425, "cost": 0.0625295, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9856, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5632, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0625295, "gateway_cost": 0.0625295 }, "finish_reason": "tool_calls", "elapsed_seconds": 53.399, "completed_at": "2026-09-07T02:36:42.876102+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 10768, "completion_tokens": 6838, "total_tokens": 17606, "cost": 0.07032, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 10240, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4736, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07032, "gateway_cost": 0.07032 }, "finish_reason": "tool_calls", "elapsed_seconds": 56.715, "completed_at": "2026-09-07T02:37:39.595628+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "787aa21f2cdfad8185b74fb871d578db08e9a6475cabff57c52cfab11b0b8139", "bytes": 952 }, { "path": "artifacts/prospects-v2.csv", "sha256": "d1a9b5a94be46df53f2a95f65f6187f88b85763934287e378984ac6998692c79", "bytes": 828 }, { "path": "artifacts/prospects-v3.csv", "sha256": "1baeaab6eeea994376ef63645ffb569aeaaea98b76d07ab7ab409830d3c6cbef", "bytes": 830 }, { "path": "artifacts/prospects-v4.csv", "sha256": "d1a9b5a94be46df53f2a95f65f6187f88b85763934287e378984ac6998692c79", "bytes": 828 }, { "path": "artifacts/prospects-v5.csv", "sha256": "787aa21f2cdfad8185b74fb871d578db08e9a6475cabff57c52cfab11b0b8139", "bytes": 952 }, { "path": "artifacts/prospects-v6.csv", "sha256": "1baeaab6eeea994376ef63645ffb569aeaaea98b76d07ab7ab409830d3c6cbef", "bytes": 830 }, { "path": "artifacts/prospects-v7.csv", "sha256": "d1a9b5a94be46df53f2a95f65f6187f88b85763934287e378984ac6998692c79", "bytes": 828 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-alder", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R6774", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6774" ], "reported_sources": [ "S-R6774" ] }, { "record_id": "R4598", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4598" ], "reported_sources": [ "S-R4598" ] }, { "record_id": "R7567", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 17640000, "expected": 17640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7567" ], "reported_sources": [ "S-R7567" ] }, { "record_id": "R8446", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 18720000, "expected": 18720000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8446" ], "reported_sources": [ "S-R8446" ] }, { "record_id": "R2663", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 19800000, "expected": 19800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2663" ], "reported_sources": [ "S-R2663" ] }, { "record_id": "R3927", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 20880000, "expected": 20880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3927" ], "reported_sources": [ "S-R3927" ] } ], "decision_checks": [ { "record_id": "R2193", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6607", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5582", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R9859", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4360", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6400", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R8446", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2682", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R1514", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8411", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2324", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R2663", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6902", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7567", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6774", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3086", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R4598", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1100", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v7.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R6774 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R4598 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7567 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8446 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8446 / contact_email: artifact differs from answer or lacks explicit unknown", "R2663 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3927 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R6774", "R4598", "R7567", "R8446", "R2663", "R3927" ], "answer_record_order": [ "R6774", "R4598", "R7567", "R8446", "R2663", "R3927" ], "field_checks": [ { "record_id": "R6774", "field": "entity_id", "actual": "ALDER-REF-61562", "original": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "record_id": "R6774", "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "original": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "record_id": "R6774", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R6774", "field": "original_principal_usd", "actual": "14400000", "original": "14400000", "expected": 14400000, "passed": true }, { "record_id": "R6774", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R6774", "field": "contact_name", "actual": "Casey Alder1", "original": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "record_id": "R6774", "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "original": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true }, { "record_id": "R4598", "field": "entity_id", "actual": "ALDER-REF-82017", "original": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "record_id": "R4598", "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "original": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "record_id": "R4598", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R4598", "field": "original_principal_usd", "actual": "16560000", "original": "16560000", "expected": 16560000, "passed": true }, { "record_id": "R4598", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R4598", "field": "contact_name", "actual": "Avery Alder3", "original": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "record_id": "R4598", "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "original": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true }, { "record_id": "R7567", "field": "entity_id", "actual": "ALDER-REF-70342", "original": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "record_id": "R7567", "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "original": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "record_id": "R7567", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R7567", "field": "original_principal_usd", "actual": "17640000", "original": "17640000", "expected": 17640000, "passed": true }, { "record_id": "R7567", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R7567", "field": "contact_name", "actual": "Sam Alder4", "original": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "record_id": "R7567", "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "original": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true }, { "record_id": "R8446", "field": "entity_id", "actual": "ALDER-REF-87904", "original": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "record_id": "R8446", "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "original": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "record_id": "R8446", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R8446", "field": "original_principal_usd", "actual": "18720000", "original": "18720000", "expected": 18720000, "passed": true }, { "record_id": "R8446", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R8446", "field": "contact_name", "actual": "Casey Alder5", "original": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "record_id": "R8446", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R2663", "field": "entity_id", "actual": "ALDER-REF-13745", "original": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "record_id": "R2663", "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "original": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "record_id": "R2663", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R2663", "field": "original_principal_usd", "actual": "19800000", "original": "19800000", "expected": 19800000, "passed": true }, { "record_id": "R2663", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R2663", "field": "contact_name", "actual": "Riley Alder6", "original": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "record_id": "R2663", "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "original": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true }, { "record_id": "R3927", "field": "entity_id", "actual": "ALDER-REF-56841", "original": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "record_id": "R3927", "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "original": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "record_id": "R3927", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R3927", "field": "original_principal_usd", "actual": "20880000", "original": "20880000", "expected": 20880000, "passed": true }, { "record_id": "R3927", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-alder/turn-07/tool-01.json" }, { "record_id": "R3927", "field": "contact_name", "actual": "Avery Alder7", "original": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "record_id": "R3927", "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "original": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R6774", "passed": true, "expected": [ "S-R6774" ], "actual": [ "S-R6774" ] }, { "record_id": "R4598", "passed": true, "expected": [ "S-R4598" ], "actual": [ "S-R4598" ] }, { "record_id": "R7567", "passed": true, "expected": [ "S-R7567" ], "actual": [ "S-R7567" ] }, { "record_id": "R8446", "passed": true, "expected": [ "S-R8446" ], "actual": [ "S-R8446" ] }, { "record_id": "R2663", "passed": true, "expected": [ "S-R2663" ], "actual": [ "S-R2663" ] }, { "record_id": "R3927", "passed": true, "expected": [ "S-R3927" ], "actual": [ "S-R3927" ] } ], "source_sha256": "d1a9b5a94be46df53f2a95f65f6187f88b85763934287e378984ac6998692c79" } }, { "case_id": "refinance-leads-alder", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:30:05.510642+00:00", "completed_at": "2026-09-07T02:31:13.881574+00:00", "elapsed_seconds": 68.371, "cost_usd": 0.29629500000000003, "known_partial_cost_usd": 0.29629500000000003, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12213, "completion_tokens": 1952, "total_tokens": 14165, "cost": 0.109865, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1169, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.109865, "gateway_cost": 0.109865 }, "finish_reason": "tool_calls", "elapsed_seconds": 20.323, "completed_at": "2026-09-07T02:30:25.851525+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13096, "completion_tokens": 4838, "total_tokens": 17934, "cost": 0.18643, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 671, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.18643, "gateway_cost": 0.18643 }, "finish_reason": "tool_calls", "elapsed_seconds": 48.024, "completed_at": "2026-09-07T02:31:13.879343+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "8fa39741b1bd00b532a7e62b617deab78ff13d2125417ea3683666e3960f2603", "bytes": 1265 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-alder", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R1514", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1514" ], "reported_sources": [ "S-R1514", "S-R6774" ] }, { "record_id": "R4598", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4598" ], "reported_sources": [ "S-R4598" ] }, { "record_id": "R7567", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 17640000, "expected": 17640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7567" ], "reported_sources": [ "S-R7567" ] }, { "record_id": "R8446", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 18720000, "expected": 18720000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8446" ], "reported_sources": [ "S-R8446" ] }, { "record_id": "R2663", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 19800000, "expected": 19800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2663" ], "reported_sources": [ "S-R2663" ] }, { "record_id": "R3927", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 20880000, "expected": 20880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3927" ], "reported_sources": [ "S-R3927" ] } ], "decision_checks": [ { "record_id": "R2193", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6607", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5582", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R9859", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4360", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6400", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R8446", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2682", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R1514", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8411", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2324", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R2663", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6902", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7567", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6774", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3086", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R4598", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1100", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R1514 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R4598 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7567 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8446 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8446 / contact_email: artifact differs from answer or lacks explicit unknown", "R2663 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3927 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R1514", "R4598", "R7567", "R8446", "R2663", "R3927" ], "answer_record_order": [ "R1514", "R4598", "R7567", "R8446", "R2663", "R3927" ], "field_checks": [ { "record_id": "R1514", "field": "entity_id", "actual": "ALDER-REF-61562", "original": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "record_id": "R1514", "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "original": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "record_id": "R1514", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R1514", "field": "original_principal_usd", "actual": "14400000", "original": "14400000", "expected": 14400000, "passed": true }, { "record_id": "R1514", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R1514", "field": "contact_name", "actual": "Casey Alder1", "original": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "record_id": "R1514", "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "original": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true }, { "record_id": "R4598", "field": "entity_id", "actual": "ALDER-REF-82017", "original": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "record_id": "R4598", "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "original": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "record_id": "R4598", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R4598", "field": "original_principal_usd", "actual": "16560000", "original": "16560000", "expected": 16560000, "passed": true }, { "record_id": "R4598", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R4598", "field": "contact_name", "actual": "Avery Alder3", "original": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "record_id": "R4598", "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "original": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true }, { "record_id": "R7567", "field": "entity_id", "actual": "ALDER-REF-70342", "original": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "record_id": "R7567", "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "original": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "record_id": "R7567", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R7567", "field": "original_principal_usd", "actual": "17640000", "original": "17640000", "expected": 17640000, "passed": true }, { "record_id": "R7567", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R7567", "field": "contact_name", "actual": "Sam Alder4", "original": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "record_id": "R7567", "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "original": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true }, { "record_id": "R8446", "field": "entity_id", "actual": "ALDER-REF-87904", "original": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "record_id": "R8446", "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "original": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "record_id": "R8446", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R8446", "field": "original_principal_usd", "actual": "18720000", "original": "18720000", "expected": 18720000, "passed": true }, { "record_id": "R8446", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R8446", "field": "contact_name", "actual": "Casey Alder5", "original": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "record_id": "R8446", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R2663", "field": "entity_id", "actual": "ALDER-REF-13745", "original": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "record_id": "R2663", "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "original": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "record_id": "R2663", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R2663", "field": "original_principal_usd", "actual": "19800000", "original": "19800000", "expected": 19800000, "passed": true }, { "record_id": "R2663", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R2663", "field": "contact_name", "actual": "Riley Alder6", "original": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "record_id": "R2663", "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "original": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true }, { "record_id": "R3927", "field": "entity_id", "actual": "ALDER-REF-56841", "original": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "record_id": "R3927", "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "original": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "record_id": "R3927", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R3927", "field": "original_principal_usd", "actual": "20880000", "original": "20880000", "expected": 20880000, "passed": true }, { "record_id": "R3927", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-alder/turn-01/tool-01.json" }, { "record_id": "R3927", "field": "contact_name", "actual": "Avery Alder7", "original": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "record_id": "R3927", "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "original": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R1514", "passed": true, "expected": [ "S-R1514", "S-R6774" ], "actual": [ "S-R1514", "S-R6774" ] }, { "record_id": "R4598", "passed": true, "expected": [ "S-R4598" ], "actual": [ "S-R4598" ] }, { "record_id": "R7567", "passed": true, "expected": [ "S-R7567" ], "actual": [ "S-R7567" ] }, { "record_id": "R8446", "passed": true, "expected": [ "S-R8446" ], "actual": [ "S-R8446" ] }, { "record_id": "R2663", "passed": true, "expected": [ "S-R2663" ], "actual": [ "S-R2663" ] }, { "record_id": "R3927", "passed": true, "expected": [ "S-R3927" ], "actual": [ "S-R3927" ] } ], "source_sha256": "8fa39741b1bd00b532a7e62b617deab78ff13d2125417ea3683666e3960f2603" } }, { "case_id": "refinance-leads-alder", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:30:05.510697+00:00", "completed_at": "2026-09-07T02:31:21.655141+00:00", "elapsed_seconds": 76.144, "cost_usd": 0.0896075, "known_partial_cost_usd": 0.0896075, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7038, "completion_tokens": 8081, "total_tokens": 15119, "cost": 0.0896075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5504, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0896075, "gateway_cost": 0.0896075 }, "finish_reason": "stop", "elapsed_seconds": 76.122, "completed_at": "2026-09-07T02:31:21.650202+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/refinance-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-alder", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 5, "verified_count": 5, "eligible_yield": 0.8333333333333334, "verified_yield": 0.8333333333333334, "selection_precision": 0.8333333333333334, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 18, "total": 20 }, "eligible_recall": { "passed": 6, "total": 7 }, "ineligible_rejection": { "passed": 12, "total": 13 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [ { "record_id": "R6607", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R6774", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6774" ], "reported_sources": [ "S-R6774" ] }, { "record_id": "R7567", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 17640000, "expected": 17640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7567" ], "reported_sources": [ "S-R7567" ] }, { "record_id": "R6607", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-28745", "expected": "ALDER-REF-28745", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-28745", "expected": "O-ALDER-REF-28745", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 17640000, "expected": 17640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Alder18", "expected": "Riley Alder18", "passed": true }, { "field": "contact_email", "actual": "finance18@refinance-leads-alder.example", "expected": "finance18@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6607" ], "reported_sources": [ "S-R6607" ] }, { "record_id": "R8446", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 18720000, "expected": 18720000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8446" ], "reported_sources": [ "S-R8446" ] }, { "record_id": "R2663", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 19800000, "expected": 19800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2663" ], "reported_sources": [ "S-R2663" ] }, { "record_id": "R3927", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 20880000, "expected": 20880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3927" ], "reported_sources": [ "S-R3927" ] } ], "decision_checks": [ { "record_id": "R2193", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6607", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5582", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R9859", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4360", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6400", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R8446", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2682", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R1514", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8411", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2324", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R2663", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6902", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7567", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6774", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3086", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R4598", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R1100", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-alder", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:31:13.882754+00:00", "completed_at": "2026-09-07T02:32:15.740825+00:00", "elapsed_seconds": 61.858, "cost_usd": 0.21089, "known_partial_cost_usd": 0.21089, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9928, "completion_tokens": 6450, "total_tokens": 16378, "cost": 0.21089, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1162, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.21089, "gateway_cost": 0.21089 }, "finish_reason": "stop", "elapsed_seconds": 61.84, "completed_at": "2026-09-07T02:32:15.740030+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/refinance-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-alder", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R1514", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-61562", "expected": "ALDER-REF-61562", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-61562", "expected": "O-ALDER-REF-61562", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder1", "expected": "Casey Alder1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-alder.example", "expected": "finance1@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1514" ], "reported_sources": [ "S-R1514", "S-R6774" ] }, { "record_id": "R4598", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-82017", "expected": "ALDER-REF-82017", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-82017", "expected": "O-ALDER-REF-82017", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder3", "expected": "Avery Alder3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-alder.example", "expected": "finance3@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4598" ], "reported_sources": [ "S-R4598" ] }, { "record_id": "R7567", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-70342", "expected": "ALDER-REF-70342", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-70342", "expected": "O-ALDER-REF-70342", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 17640000, "expected": 17640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Alder4", "expected": "Sam Alder4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-alder.example", "expected": "finance4@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7567" ], "reported_sources": [ "S-R7567" ] }, { "record_id": "R8446", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-87904", "expected": "ALDER-REF-87904", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-87904", "expected": "O-ALDER-REF-87904", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 18720000, "expected": 18720000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Alder5", "expected": "Casey Alder5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8446" ], "reported_sources": [ "S-R8446" ] }, { "record_id": "R2663", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-13745", "expected": "ALDER-REF-13745", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-13745", "expected": "O-ALDER-REF-13745", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 19800000, "expected": 19800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Alder6", "expected": "Riley Alder6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-alder.example", "expected": "finance6@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2663" ], "reported_sources": [ "S-R2663" ] }, { "record_id": "R3927", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REF-56841", "expected": "ALDER-REF-56841", "passed": true }, { "field": "owner_entity_id", "actual": "O-ALDER-REF-56841", "expected": "O-ALDER-REF-56841", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 20880000, "expected": 20880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Alder7", "expected": "Avery Alder7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-alder.example", "expected": "finance7@refinance-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3927" ], "reported_sources": [ "S-R3927" ] } ], "decision_checks": [ { "record_id": "R2193", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6607", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5582", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R9859", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4360", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6400", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R8446", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2682", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R1514", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8411", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2324", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R2663", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6902", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7567", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6774", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3086", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R4598", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1100", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-birch", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:30:44.542Z", "observed_completed_at": "2026-09-07T02:33:17.703Z", "packet_sha256": "38e1fb070030995690ce4692a7c1ffd9794d562c8e048c5df857cd1a08c68e5c", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/refinance_leads_birch_phoenix.csv", "sha256": "85e17acc1437846e5c8c948b14d2e1b6725bc94a56fc94ed494514486a5adc1e", "bytes": 2357 }, { "path": "artifacts/refinance_leads_birch_phoenix.xlsx", "sha256": "02d6e179c8e3e9ec2722f1fadcf6cb0e0b0e4d1dbc9c97176de1387d4b05c3b7", "bytes": 5895 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.41710549999999996, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 100.58, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "76d60bb6-fe06-49f0-b70e-f4ab85adcc3f", "session_id": "c249a3cf-3bda-4e89-a5be-17d87a40bcc7", "started_at": "2026-09-06T22:31:42.949000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.26349575, 0.15360975 ], "total_cost_usd": 0.41710549999999996, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 100.58, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/refinance-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-birch", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 18, "total": 20 }, "eligible_recall": { "passed": 6, "total": 7 }, "ineligible_rejection": { "passed": 12, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R3130", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R7849", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 5760000, "expected": 5760000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7849" ], "reported_sources": [ "S-R7849", "S-R8920" ] }, { "record_id": "R2258", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 6624000, "expected": 6624000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2258" ], "reported_sources": [ "S-R2258" ] }, { "record_id": "R3130", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-46281", "expected": "BIRCH-REF-46281", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-46281", "expected": "O-BIRCH-REF-46281", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 7056000, "expected": 7056000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Birch18", "expected": "Riley Birch18", "passed": true }, { "field": "contact_email", "actual": "finance18@refinance-leads-birch.example", "expected": "finance18@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3130" ], "reported_sources": [ "S-R3130" ] }, { "record_id": "R8680", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 7056000, "expected": 7056000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8680" ], "reported_sources": [ "S-R8680" ] }, { "record_id": "R9318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 7488000, "expected": 7488000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9318" ], "reported_sources": [ "S-R9318" ] }, { "record_id": "R7666", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 7920000, "expected": 7920000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7666" ], "reported_sources": [ "S-R7666" ] }, { "record_id": "R8605", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 8352000, "expected": 8352000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8605" ], "reported_sources": [ "S-R8605" ] } ], "decision_checks": [ { "record_id": "R4713", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8097", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R2258", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2031", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3130", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7666", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2900", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R8605", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8827", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R9318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7439", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R6767", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R8055", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1912", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7849", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8680", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8920", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R9844", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R9018", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R4315", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/refinance_leads_birch_phoenix.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "R7849 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2258 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3130 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8680 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9318 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9318 / contact_email: artifact differs from answer or lacks explicit unknown", "R7666 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8605 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R7849", "R2258", "R3130", "R8680", "R9318", "R7666", "R8605" ], "answer_record_order": [ "R7849", "R2258", "R3130", "R8680", "R9318", "R7666", "R8605" ], "field_checks": [ { "record_id": "R7849", "field": "entity_id", "actual": "BIRCH-REF-71420", "original": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "record_id": "R7849", "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "original": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "record_id": "R7849", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R7849", "field": "original_principal_usd", "actual": "5760000", "original": "5760000", "expected": 5760000, "passed": true }, { "record_id": "R7849", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7849", "field": "contact_name", "actual": "Casey Birch1", "original": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "record_id": "R7849", "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "original": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true }, { "record_id": "R2258", "field": "entity_id", "actual": "BIRCH-REF-42124", "original": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "record_id": "R2258", "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "original": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "record_id": "R2258", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R2258", "field": "original_principal_usd", "actual": "6624000", "original": "6624000", "expected": 6624000, "passed": true }, { "record_id": "R2258", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R2258", "field": "contact_name", "actual": "Avery Birch3", "original": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "record_id": "R2258", "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "original": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true }, { "record_id": "R3130", "field": "entity_id", "actual": "BIRCH-REF-46281", "original": "BIRCH-REF-46281", "expected": "BIRCH-REF-46281", "passed": true }, { "record_id": "R3130", "field": "owner_entity_id", "actual": "O-BIRCH-REF-46281", "original": "O-BIRCH-REF-46281", "expected": "O-BIRCH-REF-46281", "passed": true }, { "record_id": "R3130", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R3130", "field": "original_principal_usd", "actual": "7056000", "original": "7056000", "expected": 7056000, "passed": true }, { "record_id": "R3130", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R3130", "field": "contact_name", "actual": "Riley Birch18", "original": "Riley Birch18", "expected": "Riley Birch18", "passed": true }, { "record_id": "R3130", "field": "contact_email", "actual": "finance18@refinance-leads-birch.example", "original": "finance18@refinance-leads-birch.example", "expected": "finance18@refinance-leads-birch.example", "passed": true }, { "record_id": "R8680", "field": "entity_id", "actual": "BIRCH-REF-10272", "original": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "record_id": "R8680", "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "original": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "record_id": "R8680", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R8680", "field": "original_principal_usd", "actual": "7056000", "original": "7056000", "expected": 7056000, "passed": true }, { "record_id": "R8680", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R8680", "field": "contact_name", "actual": "Sam Birch4", "original": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "record_id": "R8680", "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "original": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true }, { "record_id": "R9318", "field": "entity_id", "actual": "BIRCH-REF-75599", "original": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "record_id": "R9318", "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "original": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "record_id": "R9318", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R9318", "field": "original_principal_usd", "actual": "7488000", "original": "7488000", "expected": 7488000, "passed": true }, { "record_id": "R9318", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R9318", "field": "contact_name", "actual": "Casey Birch5", "original": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "record_id": "R9318", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7666", "field": "entity_id", "actual": "BIRCH-REF-37514", "original": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "record_id": "R7666", "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "original": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "record_id": "R7666", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R7666", "field": "original_principal_usd", "actual": "7920000", "original": "7920000", "expected": 7920000, "passed": true }, { "record_id": "R7666", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7666", "field": "contact_name", "actual": "Riley Birch6", "original": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "record_id": "R7666", "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "original": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true }, { "record_id": "R8605", "field": "entity_id", "actual": "BIRCH-REF-88672", "original": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "record_id": "R8605", "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "original": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "record_id": "R8605", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R8605", "field": "original_principal_usd", "actual": "8352000", "original": "8352000", "expected": 8352000, "passed": true }, { "record_id": "R8605", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R8605", "field": "contact_name", "actual": "Avery Birch7", "original": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "record_id": "R8605", "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "original": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R7849", "passed": true, "expected": [ "S-R7849", "S-R8920" ], "actual": [ "S-R7849", "S-R8920" ] }, { "record_id": "R2258", "passed": true, "expected": [ "S-R2258" ], "actual": [ "S-R2258", "U-R2258-2" ] }, { "record_id": "R3130", "passed": true, "expected": [ "S-R3130" ], "actual": [ "S-R3130" ] }, { "record_id": "R8680", "passed": true, "expected": [ "S-R8680" ], "actual": [ "S-R8680", "U-R8680-3" ] }, { "record_id": "R9318", "passed": true, "expected": [ "S-R9318" ], "actual": [ "S-R9318" ] }, { "record_id": "R7666", "passed": true, "expected": [ "S-R7666" ], "actual": [ "S-R7666" ] }, { "record_id": "R8605", "passed": true, "expected": [ "S-R8605" ], "actual": [ "S-R8605" ] } ], "source_sha256": "85e17acc1437846e5c8c948b14d2e1b6725bc94a56fc94ed494514486a5adc1e" } }, { "case_id": "refinance-leads-birch", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:31:21.656461+00:00", "completed_at": "2026-09-07T02:36:18.864828+00:00", "elapsed_seconds": 297.207, "cost_usd": 0.33343325, "known_partial_cost_usd": 0.33343325, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 5, "tool_calls": 5, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7808, "completion_tokens": 6852, "total_tokens": 14660, "cost": 0.07828, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6336, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07828, "gateway_cost": 0.07828 }, "finish_reason": "tool_calls", "elapsed_seconds": 73.561, "completed_at": "2026-09-07T02:32:35.238153+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8288, "completion_tokens": 4185, "total_tokens": 12473, "cost": 0.04357, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7680, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3776, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.04357, "gateway_cost": 0.04357 }, "finish_reason": "tool_calls", "elapsed_seconds": 38.311, "completed_at": "2026-09-07T02:33:13.552858+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8709, "completion_tokens": 5468, "total_tokens": 14177, "cost": 0.05635025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8192, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4992, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05635025, "gateway_cost": 0.05635025 }, "finish_reason": "tool_calls", "elapsed_seconds": 43.363, "completed_at": "2026-09-07T02:33:56.919984+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9189, "completion_tokens": 6830, "total_tokens": 16019, "cost": 0.07013825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8576, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6400, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07013825, "gateway_cost": 0.07013825 }, "finish_reason": "tool_calls", "elapsed_seconds": 66.072, "completed_at": "2026-09-07T02:35:02.996239+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9615, "completion_tokens": 8330, "total_tokens": 17945, "cost": 0.08509475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9088, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6336, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08509475, "gateway_cost": 0.08509475 }, "finish_reason": "tool_calls", "elapsed_seconds": 75.859, "completed_at": "2026-09-07T02:36:18.860318+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "628c4841fd2e175320d1af74296110824dfced0235bd4cf37a3181849dce3689", "bytes": 945 }, { "path": "artifacts/prospects-v2.csv", "sha256": "41a565359d510a8f9fb65252785d9ed333f1b8427567d31a6070e6f1388fdef7", "bytes": 822 }, { "path": "artifacts/prospects-v3.csv", "sha256": "d5d7fd879374eda4daa03d72328d8a8c932cfe966f61ac208197f13dfa3b944b", "bytes": 945 }, { "path": "artifacts/prospects-v4.csv", "sha256": "82dca5f5cb573d465ff547048e2b9c2a9f76f73b8dd5c6a947d8560e6ec009f9", "bytes": 831 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-birch", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R8920", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 5760000, "expected": 5760000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8920" ], "reported_sources": [ "S-R8920", "S-R7849" ] }, { "record_id": "R2258", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 6624000, "expected": 6624000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2258" ], "reported_sources": [ "S-R2258" ] }, { "record_id": "R8680", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 7056000, "expected": 7056000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8680" ], "reported_sources": [ "S-R8680" ] }, { "record_id": "R9318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 7488000, "expected": 7488000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9318" ], "reported_sources": [ "S-R9318" ] }, { "record_id": "R7666", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 7920000, "expected": 7920000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7666" ], "reported_sources": [ "S-R7666" ] }, { "record_id": "R8605", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 8352000, "expected": 8352000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8605" ], "reported_sources": [ "S-R8605" ] } ], "decision_checks": [ { "record_id": "R4713", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8097", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R2258", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2031", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7666", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2900", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R8605", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8827", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R9318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7439", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R6767", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R8055", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1912", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7849", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8680", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8920", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9844", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R9018", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R4315", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v4.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R8920 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2258 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8680 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9318 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9318 / contact_email: artifact differs from answer or lacks explicit unknown", "R7666 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8605 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R8920", "R2258", "R8680", "R9318", "R7666", "R8605" ], "answer_record_order": [ "R8920", "R2258", "R8680", "R9318", "R7666", "R8605" ], "field_checks": [ { "record_id": "R8920", "field": "entity_id", "actual": "BIRCH-REF-71420", "original": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "record_id": "R8920", "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "original": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "record_id": "R8920", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R8920", "field": "original_principal_usd", "actual": "5760000", "original": "5760000", "expected": 5760000, "passed": true }, { "record_id": "R8920", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R8920", "field": "contact_name", "actual": "Casey Birch1", "original": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "record_id": "R8920", "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "original": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true }, { "record_id": "R2258", "field": "entity_id", "actual": "BIRCH-REF-42124", "original": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "record_id": "R2258", "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "original": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "record_id": "R2258", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R2258", "field": "original_principal_usd", "actual": "6624000", "original": "6624000", "expected": 6624000, "passed": true }, { "record_id": "R2258", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R2258", "field": "contact_name", "actual": "Avery Birch3", "original": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "record_id": "R2258", "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "original": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true }, { "record_id": "R8680", "field": "entity_id", "actual": "BIRCH-REF-10272", "original": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "record_id": "R8680", "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "original": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "record_id": "R8680", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R8680", "field": "original_principal_usd", "actual": "7056000", "original": "7056000", "expected": 7056000, "passed": true }, { "record_id": "R8680", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R8680", "field": "contact_name", "actual": "Sam Birch4", "original": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "record_id": "R8680", "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "original": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true }, { "record_id": "R9318", "field": "entity_id", "actual": "BIRCH-REF-75599", "original": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "record_id": "R9318", "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "original": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "record_id": "R9318", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R9318", "field": "original_principal_usd", "actual": "7488000", "original": "7488000", "expected": 7488000, "passed": true }, { "record_id": "R9318", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R9318", "field": "contact_name", "actual": "Casey Birch5", "original": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "record_id": "R9318", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R7666", "field": "entity_id", "actual": "BIRCH-REF-37514", "original": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "record_id": "R7666", "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "original": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "record_id": "R7666", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R7666", "field": "original_principal_usd", "actual": "7920000", "original": "7920000", "expected": 7920000, "passed": true }, { "record_id": "R7666", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R7666", "field": "contact_name", "actual": "Riley Birch6", "original": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "record_id": "R7666", "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "original": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true }, { "record_id": "R8605", "field": "entity_id", "actual": "BIRCH-REF-88672", "original": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "record_id": "R8605", "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "original": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "record_id": "R8605", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R8605", "field": "original_principal_usd", "actual": "8352000", "original": "8352000", "expected": 8352000, "passed": true }, { "record_id": "R8605", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-birch/turn-04/tool-01.json" }, { "record_id": "R8605", "field": "contact_name", "actual": "Avery Birch7", "original": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "record_id": "R8605", "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "original": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R8920", "passed": true, "expected": [ "S-R7849", "S-R8920" ], "actual": [ "S-R7849", "S-R8920" ] }, { "record_id": "R2258", "passed": true, "expected": [ "S-R2258" ], "actual": [ "S-R2258" ] }, { "record_id": "R8680", "passed": true, "expected": [ "S-R8680" ], "actual": [ "S-R8680" ] }, { "record_id": "R9318", "passed": true, "expected": [ "S-R9318" ], "actual": [ "S-R9318" ] }, { "record_id": "R7666", "passed": true, "expected": [ "S-R7666" ], "actual": [ "S-R7666" ] }, { "record_id": "R8605", "passed": true, "expected": [ "S-R8605" ], "actual": [ "S-R8605" ] } ], "source_sha256": "82dca5f5cb573d465ff547048e2b9c2a9f76f73b8dd5c6a947d8560e6ec009f9" } }, { "case_id": "refinance-leads-birch", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:32:15.742473+00:00", "completed_at": "2026-09-07T02:33:18.644299+00:00", "elapsed_seconds": 62.902, "cost_usd": 0.288935, "known_partial_cost_usd": 0.288935, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12252, "completion_tokens": 1850, "total_tokens": 14102, "cost": 0.10751, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1017, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10751, "gateway_cost": 0.10751 }, "finish_reason": "tool_calls", "elapsed_seconds": 18.142, "completed_at": "2026-09-07T02:32:33.902633+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13185, "completion_tokens": 4620, "total_tokens": 17805, "cost": 0.181425, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 537, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.181425, "gateway_cost": 0.181425 }, "finish_reason": "tool_calls", "elapsed_seconds": 44.736, "completed_at": "2026-09-07T02:33:18.642019+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "ce3a7d2415464ed92463dceb6ed377e1ab74a8ee056911b7743f7c08a2b9e103", "bytes": 1326 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-birch", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R8920", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 5760000, "expected": 5760000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8920" ], "reported_sources": [ "S-R8920", "S-R7849" ] }, { "record_id": "R2258", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 6624000, "expected": 6624000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2258" ], "reported_sources": [ "S-R2258" ] }, { "record_id": "R8680", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 7056000, "expected": 7056000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8680" ], "reported_sources": [ "S-R8680" ] }, { "record_id": "R9318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 7488000, "expected": 7488000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9318" ], "reported_sources": [ "S-R9318" ] }, { "record_id": "R7666", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 7920000, "expected": 7920000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7666" ], "reported_sources": [ "S-R7666" ] }, { "record_id": "R8605", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 8352000, "expected": 8352000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8605" ], "reported_sources": [ "S-R8605" ] } ], "decision_checks": [ { "record_id": "R4713", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8097", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R2258", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2031", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7666", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2900", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R8605", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8827", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R9318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7439", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R6767", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R8055", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1912", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7849", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8680", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8920", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9844", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R9018", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R4315", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R8920 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2258 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8680 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9318 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9318 / contact_email: artifact differs from answer or lacks explicit unknown", "R7666 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8605 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R8920", "R2258", "R8680", "R9318", "R7666", "R8605" ], "answer_record_order": [ "R8920", "R2258", "R8680", "R9318", "R7666", "R8605" ], "field_checks": [ { "record_id": "R8920", "field": "entity_id", "actual": "BIRCH-REF-71420", "original": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "record_id": "R8920", "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "original": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "record_id": "R8920", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R8920", "field": "original_principal_usd", "actual": "5760000", "original": "5760000", "expected": 5760000, "passed": true }, { "record_id": "R8920", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R8920", "field": "contact_name", "actual": "Casey Birch1", "original": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "record_id": "R8920", "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "original": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true }, { "record_id": "R2258", "field": "entity_id", "actual": "BIRCH-REF-42124", "original": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "record_id": "R2258", "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "original": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "record_id": "R2258", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R2258", "field": "original_principal_usd", "actual": "6624000", "original": "6624000", "expected": 6624000, "passed": true }, { "record_id": "R2258", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R2258", "field": "contact_name", "actual": "Avery Birch3", "original": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "record_id": "R2258", "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "original": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true }, { "record_id": "R8680", "field": "entity_id", "actual": "BIRCH-REF-10272", "original": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "record_id": "R8680", "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "original": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "record_id": "R8680", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R8680", "field": "original_principal_usd", "actual": "7056000", "original": "7056000", "expected": 7056000, "passed": true }, { "record_id": "R8680", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R8680", "field": "contact_name", "actual": "Sam Birch4", "original": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "record_id": "R8680", "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "original": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true }, { "record_id": "R9318", "field": "entity_id", "actual": "BIRCH-REF-75599", "original": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "record_id": "R9318", "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "original": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "record_id": "R9318", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R9318", "field": "original_principal_usd", "actual": "7488000", "original": "7488000", "expected": 7488000, "passed": true }, { "record_id": "R9318", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R9318", "field": "contact_name", "actual": "Casey Birch5", "original": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "record_id": "R9318", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R7666", "field": "entity_id", "actual": "BIRCH-REF-37514", "original": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "record_id": "R7666", "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "original": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "record_id": "R7666", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R7666", "field": "original_principal_usd", "actual": "7920000", "original": "7920000", "expected": 7920000, "passed": true }, { "record_id": "R7666", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R7666", "field": "contact_name", "actual": "Riley Birch6", "original": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "record_id": "R7666", "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "original": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true }, { "record_id": "R8605", "field": "entity_id", "actual": "BIRCH-REF-88672", "original": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "record_id": "R8605", "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "original": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "record_id": "R8605", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R8605", "field": "original_principal_usd", "actual": "8352000", "original": "8352000", "expected": 8352000, "passed": true }, { "record_id": "R8605", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-birch/turn-01/tool-01.json" }, { "record_id": "R8605", "field": "contact_name", "actual": "Avery Birch7", "original": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "record_id": "R8605", "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "original": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R8920", "passed": true, "expected": [ "S-R7849", "S-R8920" ], "actual": [ "S-R7849", "S-R8920" ] }, { "record_id": "R2258", "passed": true, "expected": [ "S-R2258" ], "actual": [ "S-R2258", "U-R2258-2" ] }, { "record_id": "R8680", "passed": true, "expected": [ "S-R8680" ], "actual": [ "S-R8680", "U-R8680-3" ] }, { "record_id": "R9318", "passed": true, "expected": [ "S-R9318" ], "actual": [ "S-R9318" ] }, { "record_id": "R7666", "passed": true, "expected": [ "S-R7666" ], "actual": [ "S-R7666" ] }, { "record_id": "R8605", "passed": true, "expected": [ "S-R8605" ], "actual": [ "S-R8605" ] } ], "source_sha256": "ce3a7d2415464ed92463dceb6ed377e1ab74a8ee056911b7743f7c08a2b9e103" } }, { "case_id": "refinance-leads-birch", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:33:18.645301+00:00", "completed_at": "2026-09-07T02:34:18.012909+00:00", "elapsed_seconds": 59.367, "cost_usd": 0.0954975, "known_partial_cost_usd": 0.0954975, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7062, "completion_tokens": 8667, "total_tokens": 15729, "cost": 0.0954975, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6016, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0954975, "gateway_cost": 0.0954975 }, "finish_reason": "stop", "elapsed_seconds": 59.341, "completed_at": "2026-09-07T02:34:18.010755+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/refinance-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-birch", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R8920", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 5760000, "expected": 5760000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8920" ], "reported_sources": [ "S-R8920" ] }, { "record_id": "R2258", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 6624000, "expected": 6624000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2258" ], "reported_sources": [ "S-R2258" ] }, { "record_id": "R8680", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 7056000, "expected": 7056000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8680" ], "reported_sources": [ "S-R8680" ] }, { "record_id": "R9318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 7488000, "expected": 7488000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9318" ], "reported_sources": [ "S-R9318" ] }, { "record_id": "R7666", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 7920000, "expected": 7920000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7666" ], "reported_sources": [ "S-R7666" ] }, { "record_id": "R8605", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 8352000, "expected": 8352000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8605" ], "reported_sources": [ "S-R8605" ] } ], "decision_checks": [ { "record_id": "R4713", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8097", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R2258", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2031", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7666", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2900", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R8605", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8827", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R9318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7439", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R6767", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R8055", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1912", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7849", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8680", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8920", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9844", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R9018", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R4315", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-birch", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:34:18.014399+00:00", "completed_at": "2026-09-07T02:35:17.001277+00:00", "elapsed_seconds": 58.987, "cost_usd": 0.20736, "known_partial_cost_usd": 0.20736, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9967, "completion_tokens": 6301, "total_tokens": 16268, "cost": 0.20736, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 984, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.20736, "gateway_cost": 0.20736 }, "finish_reason": "stop", "elapsed_seconds": 58.968, "completed_at": "2026-09-07T02:35:16.999551+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/refinance-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-birch", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R8920", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-71420", "expected": "BIRCH-REF-71420", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-71420", "expected": "O-BIRCH-REF-71420", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 5760000, "expected": 5760000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch1", "expected": "Casey Birch1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-birch.example", "expected": "finance1@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8920" ], "reported_sources": [ "S-R8920" ] }, { "record_id": "R2258", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-42124", "expected": "BIRCH-REF-42124", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-42124", "expected": "O-BIRCH-REF-42124", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 6624000, "expected": 6624000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch3", "expected": "Avery Birch3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-birch.example", "expected": "finance3@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2258" ], "reported_sources": [ "S-R2258" ] }, { "record_id": "R8680", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-10272", "expected": "BIRCH-REF-10272", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-10272", "expected": "O-BIRCH-REF-10272", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 7056000, "expected": 7056000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Birch4", "expected": "Sam Birch4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-birch.example", "expected": "finance4@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8680" ], "reported_sources": [ "S-R8680" ] }, { "record_id": "R9318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-75599", "expected": "BIRCH-REF-75599", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-75599", "expected": "O-BIRCH-REF-75599", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 7488000, "expected": 7488000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Birch5", "expected": "Casey Birch5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9318" ], "reported_sources": [ "S-R9318" ] }, { "record_id": "R7666", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-37514", "expected": "BIRCH-REF-37514", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-37514", "expected": "O-BIRCH-REF-37514", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 7920000, "expected": 7920000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Birch6", "expected": "Riley Birch6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-birch.example", "expected": "finance6@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7666" ], "reported_sources": [ "S-R7666" ] }, { "record_id": "R8605", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REF-88672", "expected": "BIRCH-REF-88672", "passed": true }, { "field": "owner_entity_id", "actual": "O-BIRCH-REF-88672", "expected": "O-BIRCH-REF-88672", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 8352000, "expected": 8352000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Birch7", "expected": "Avery Birch7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-birch.example", "expected": "finance7@refinance-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8605" ], "reported_sources": [ "S-R8605" ] } ], "decision_checks": [ { "record_id": "R4713", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8097", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R2258", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2031", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7666", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2900", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R8605", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8827", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R9318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7439", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R6767", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R8055", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1912", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7849", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8680", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8920", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9844", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R9018", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R4315", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-cypress", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:38:07.018Z", "observed_completed_at": "2026-09-07T02:40:12.762Z", "packet_sha256": "429d0399e73da8cfa26dd89704f344ea2d5f33b82bd1868d349be0e65b0497da", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/refinance_leads_cypress_prospects.csv", "sha256": "f7072939d735f0e3ee0882e3b45b9c3489105cdca6ea2ee6171d89222f37c9b1", "bytes": 1018 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.34473925, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 78.64, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "95b2571b-dd6d-4bb6-b1ff-0e32ae87657f", "session_id": "64998466-6cc0-4c28-a35b-2e37022fa2b1", "started_at": "2026-09-06T22:38:49.466000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.210334, 0.13440525 ], "total_cost_usd": 0.34473925, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 78.64, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/refinance-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-cypress", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 12, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R3917", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R2362", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 4800000, "expected": 4800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2362" ], "reported_sources": [ "S-R2362" ] }, { "record_id": "R4643", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 5520000, "expected": 5520000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4643" ], "reported_sources": [ "S-R4643" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353" ], "reported_sources": [ "S-R3353" ] }, { "record_id": "R3917", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-21679", "expected": "CYPRESS-REF-21679", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-21679", "expected": "O-CYPRESS-REF-21679", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress18", "expected": "Riley Cypress18", "passed": true }, { "field": "contact_email", "actual": "finance18@refinance-leads-cypress.example", "expected": "finance18@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3917" ], "reported_sources": [ "S-R3917" ] }, { "record_id": "R2253", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 6240000, "expected": 6240000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2253" ], "reported_sources": [ "S-R2253" ] }, { "record_id": "R5546", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 6600000, "expected": 6600000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5546" ], "reported_sources": [ "S-R5546" ] }, { "record_id": "R7918", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 6960000, "expected": 6960000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7918" ], "reported_sources": [ "S-R7918" ] } ], "decision_checks": [ { "record_id": "R3947", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R3917", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3389", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7918", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2362", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9569", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R9318", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9030", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8853", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2334", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R2570", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2253", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4643", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3709", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4862", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5546", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7391", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3836", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/refinance_leads_cypress_prospects.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "R2362 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R4643 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3353 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3917 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2253 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2253 / contact_email: artifact differs from answer or lacks explicit unknown", "R5546 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7918 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2362", "R4643", "R3353", "R3917", "R2253", "R5546", "R7918" ], "answer_record_order": [ "R2362", "R4643", "R3353", "R3917", "R2253", "R5546", "R7918" ], "field_checks": [ { "record_id": "R2362", "field": "entity_id", "actual": "CYPRESS-REF-62661", "original": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "record_id": "R2362", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "original": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "record_id": "R2362", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R2362", "field": "original_principal_usd", "actual": "4800000", "original": "4800000", "expected": 4800000, "passed": true }, { "record_id": "R2362", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R2362", "field": "contact_name", "actual": "Casey Cypress1", "original": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "record_id": "R2362", "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "original": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true }, { "record_id": "R4643", "field": "entity_id", "actual": "CYPRESS-REF-10959", "original": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "record_id": "R4643", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "original": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "record_id": "R4643", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R4643", "field": "original_principal_usd", "actual": "5520000", "original": "5520000", "expected": 5520000, "passed": true }, { "record_id": "R4643", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R4643", "field": "contact_name", "actual": "Avery Cypress3", "original": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "record_id": "R4643", "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "original": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true }, { "record_id": "R3353", "field": "entity_id", "actual": "CYPRESS-REF-27033", "original": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "record_id": "R3353", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "original": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "record_id": "R3353", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R3353", "field": "original_principal_usd", "actual": "5880000", "original": "5880000", "expected": 5880000, "passed": true }, { "record_id": "R3353", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R3353", "field": "contact_name", "actual": "Sam Cypress4", "original": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "record_id": "R3353", "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "original": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true }, { "record_id": "R3917", "field": "entity_id", "actual": "CYPRESS-REF-21679", "original": "CYPRESS-REF-21679", "expected": "CYPRESS-REF-21679", "passed": true }, { "record_id": "R3917", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-21679", "original": "O-CYPRESS-REF-21679", "expected": "O-CYPRESS-REF-21679", "passed": true }, { "record_id": "R3917", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R3917", "field": "original_principal_usd", "actual": "5880000", "original": "5880000", "expected": 5880000, "passed": true }, { "record_id": "R3917", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R3917", "field": "contact_name", "actual": "Riley Cypress18", "original": "Riley Cypress18", "expected": "Riley Cypress18", "passed": true }, { "record_id": "R3917", "field": "contact_email", "actual": "finance18@refinance-leads-cypress.example", "original": "finance18@refinance-leads-cypress.example", "expected": "finance18@refinance-leads-cypress.example", "passed": true }, { "record_id": "R2253", "field": "entity_id", "actual": "CYPRESS-REF-14488", "original": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "record_id": "R2253", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "original": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "record_id": "R2253", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R2253", "field": "original_principal_usd", "actual": "6240000", "original": "6240000", "expected": 6240000, "passed": true }, { "record_id": "R2253", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R2253", "field": "contact_name", "actual": "Casey Cypress5", "original": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "record_id": "R2253", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R5546", "field": "entity_id", "actual": "CYPRESS-REF-65445", "original": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "record_id": "R5546", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "original": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "record_id": "R5546", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R5546", "field": "original_principal_usd", "actual": "6600000", "original": "6600000", "expected": 6600000, "passed": true }, { "record_id": "R5546", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R5546", "field": "contact_name", "actual": "Riley Cypress6", "original": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "record_id": "R5546", "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "original": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true }, { "record_id": "R7918", "field": "entity_id", "actual": "CYPRESS-REF-34203", "original": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "record_id": "R7918", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "original": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "record_id": "R7918", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R7918", "field": "original_principal_usd", "actual": "6960000", "original": "6960000", "expected": 6960000, "passed": true }, { "record_id": "R7918", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7918", "field": "contact_name", "actual": "Avery Cypress7", "original": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "record_id": "R7918", "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "original": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2362", "passed": true, "expected": [ "S-R2362" ], "actual": [ "S-R2362" ] }, { "record_id": "R4643", "passed": true, "expected": [ "S-R4643" ], "actual": [ "S-R4643" ] }, { "record_id": "R3353", "passed": true, "expected": [ "S-R3353" ], "actual": [ "S-R3353" ] }, { "record_id": "R3917", "passed": true, "expected": [ "S-R3917" ], "actual": [ "S-R3917" ] }, { "record_id": "R2253", "passed": true, "expected": [ "S-R2253" ], "actual": [ "S-R2253" ] }, { "record_id": "R5546", "passed": true, "expected": [ "S-R5546" ], "actual": [ "S-R5546" ] }, { "record_id": "R7918", "passed": true, "expected": [ "S-R7918" ], "actual": [ "S-R7918" ] } ], "source_sha256": "f7072939d735f0e3ee0882e3b45b9c3489105cdca6ea2ee6171d89222f37c9b1" } }, { "case_id": "refinance-leads-cypress", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:35:17.002609+00:00", "completed_at": "2026-09-07T02:38:15.424110+00:00", "elapsed_seconds": 178.421, "cost_usd": 0.2018775, "known_partial_cost_usd": 0.2018775, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7807, "completion_tokens": 9345, "total_tokens": 17152, "cost": 0.10320875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 8896, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10320875, "gateway_cost": 0.10320875 }, "finish_reason": "tool_calls", "elapsed_seconds": 95.761, "completed_at": "2026-09-07T02:36:52.789548+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8223, "completion_tokens": 9703, "total_tokens": 17926, "cost": 0.09866875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7680, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7552, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.09866875, "gateway_cost": 0.09866875 }, "finish_reason": "tool_calls", "elapsed_seconds": 82.628, "completed_at": "2026-09-07T02:38:15.421719+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "ce9dfb558547ff82d2d73ef018a7ca5de623ad6687b65ae8aeb54798ff143e4a", "bytes": 868 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-cypress", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R7391", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 4800000, "expected": 4800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7391" ], "reported_sources": [ "S-R7391" ] }, { "record_id": "R4643", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 5520000, "expected": 5520000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4643" ], "reported_sources": [ "S-R4643" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353" ], "reported_sources": [ "S-R3353" ] }, { "record_id": "R2253", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 6240000, "expected": 6240000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2253" ], "reported_sources": [ "S-R2253" ] }, { "record_id": "R5546", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 6600000, "expected": 6600000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5546" ], "reported_sources": [ "S-R5546" ] }, { "record_id": "R7918", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 6960000, "expected": 6960000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7918" ], "reported_sources": [ "S-R7918" ] } ], "decision_checks": [ { "record_id": "R3947", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R3917", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3389", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7918", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2362", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9569", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R9318", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9030", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8853", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2334", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R2570", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2253", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4643", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3709", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4862", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5546", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7391", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3836", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R7391 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R4643 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3353 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2253 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2253 / contact_email: artifact differs from answer or lacks explicit unknown", "R5546 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7918 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R7391", "R4643", "R3353", "R2253", "R5546", "R7918" ], "answer_record_order": [ "R7391", "R4643", "R3353", "R2253", "R5546", "R7918" ], "field_checks": [ { "record_id": "R7391", "field": "entity_id", "actual": "CYPRESS-REF-62661", "original": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "record_id": "R7391", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "original": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "record_id": "R7391", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R7391", "field": "original_principal_usd", "actual": "4800000", "original": "4800000", "expected": 4800000, "passed": true }, { "record_id": "R7391", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R7391", "field": "contact_name", "actual": "Casey Cypress1", "original": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "record_id": "R7391", "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "original": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true }, { "record_id": "R4643", "field": "entity_id", "actual": "CYPRESS-REF-10959", "original": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "record_id": "R4643", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "original": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "record_id": "R4643", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R4643", "field": "original_principal_usd", "actual": "5520000", "original": "5520000", "expected": 5520000, "passed": true }, { "record_id": "R4643", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R4643", "field": "contact_name", "actual": "Avery Cypress3", "original": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "record_id": "R4643", "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "original": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true }, { "record_id": "R3353", "field": "entity_id", "actual": "CYPRESS-REF-27033", "original": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "record_id": "R3353", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "original": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "record_id": "R3353", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R3353", "field": "original_principal_usd", "actual": "5880000", "original": "5880000", "expected": 5880000, "passed": true }, { "record_id": "R3353", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R3353", "field": "contact_name", "actual": "Sam Cypress4", "original": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "record_id": "R3353", "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "original": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true }, { "record_id": "R2253", "field": "entity_id", "actual": "CYPRESS-REF-14488", "original": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "record_id": "R2253", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "original": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "record_id": "R2253", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R2253", "field": "original_principal_usd", "actual": "6240000", "original": "6240000", "expected": 6240000, "passed": true }, { "record_id": "R2253", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R2253", "field": "contact_name", "actual": "Casey Cypress5", "original": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "record_id": "R2253", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R5546", "field": "entity_id", "actual": "CYPRESS-REF-65445", "original": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "record_id": "R5546", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "original": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "record_id": "R5546", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R5546", "field": "original_principal_usd", "actual": "6600000", "original": "6600000", "expected": 6600000, "passed": true }, { "record_id": "R5546", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R5546", "field": "contact_name", "actual": "Riley Cypress6", "original": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "record_id": "R5546", "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "original": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true }, { "record_id": "R7918", "field": "entity_id", "actual": "CYPRESS-REF-34203", "original": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "record_id": "R7918", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "original": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "record_id": "R7918", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R7918", "field": "original_principal_usd", "actual": "6960000", "original": "6960000", "expected": 6960000, "passed": true }, { "record_id": "R7918", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R7918", "field": "contact_name", "actual": "Avery Cypress7", "original": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "record_id": "R7918", "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "original": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R7391", "passed": true, "expected": [ "S-R7391" ], "actual": [ "S-R7391" ] }, { "record_id": "R4643", "passed": true, "expected": [ "S-R4643" ], "actual": [ "S-R4643" ] }, { "record_id": "R3353", "passed": true, "expected": [ "S-R3353" ], "actual": [ "S-R3353" ] }, { "record_id": "R2253", "passed": true, "expected": [ "S-R2253" ], "actual": [ "S-R2253" ] }, { "record_id": "R5546", "passed": true, "expected": [ "S-R5546" ], "actual": [ "S-R5546" ] }, { "record_id": "R7918", "passed": true, "expected": [ "S-R7918" ], "actual": [ "S-R7918" ] } ], "source_sha256": "ce9dfb558547ff82d2d73ef018a7ca5de623ad6687b65ae8aeb54798ff143e4a" } }, { "case_id": "refinance-leads-cypress", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:36:18.866359+00:00", "completed_at": "2026-09-07T02:37:26.582245+00:00", "elapsed_seconds": 67.716, "cost_usd": 0.29891, "known_partial_cost_usd": 0.29891, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12393, "completion_tokens": 1838, "total_tokens": 14231, "cost": 0.107915, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1048, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.107915, "gateway_cost": 0.107915 }, "finish_reason": "tool_calls", "elapsed_seconds": 19.367, "completed_at": "2026-09-07T02:36:38.250348+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13194, "completion_tokens": 5001, "total_tokens": 18195, "cost": 0.190995, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 753, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.190995, "gateway_cost": 0.190995 }, "finish_reason": "tool_calls", "elapsed_seconds": 48.325, "completed_at": "2026-09-07T02:37:26.578810+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "e1fdbefd562c4b99f422aa31bbb9f221c0c5fc325858106c976b3fc60e18de91", "bytes": 1127 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-cypress", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2362", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 4800000, "expected": 4800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2362" ], "reported_sources": [ "S-R2362", "S-R7391" ] }, { "record_id": "R4643", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 5520000, "expected": 5520000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4643" ], "reported_sources": [ "S-R4643" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353" ], "reported_sources": [ "S-R3353" ] }, { "record_id": "R2253", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 6240000, "expected": 6240000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2253" ], "reported_sources": [ "S-R2253" ] }, { "record_id": "R5546", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 6600000, "expected": 6600000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5546" ], "reported_sources": [ "S-R5546" ] }, { "record_id": "R7918", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 6960000, "expected": 6960000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7918" ], "reported_sources": [ "S-R7918" ] } ], "decision_checks": [ { "record_id": "R3947", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R3917", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3389", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7918", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2362", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9569", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R9318", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9030", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8853", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2334", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R2570", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2253", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4643", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3709", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4862", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5546", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7391", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3836", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R2362 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R4643 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R3353 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2253 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2253 / contact_email: artifact differs from answer or lacks explicit unknown", "R5546 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7918 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2362", "R4643", "R3353", "R2253", "R5546", "R7918" ], "answer_record_order": [ "R2362", "R4643", "R3353", "R2253", "R5546", "R7918" ], "field_checks": [ { "record_id": "R2362", "field": "entity_id", "actual": "CYPRESS-REF-62661", "original": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "record_id": "R2362", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "original": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "record_id": "R2362", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R2362", "field": "original_principal_usd", "actual": "4800000", "original": "4800000", "expected": 4800000, "passed": true }, { "record_id": "R2362", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R2362", "field": "contact_name", "actual": "Casey Cypress1", "original": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "record_id": "R2362", "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "original": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true }, { "record_id": "R4643", "field": "entity_id", "actual": "CYPRESS-REF-10959", "original": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "record_id": "R4643", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "original": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "record_id": "R4643", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R4643", "field": "original_principal_usd", "actual": "5520000", "original": "5520000", "expected": 5520000, "passed": true }, { "record_id": "R4643", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R4643", "field": "contact_name", "actual": "Avery Cypress3", "original": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "record_id": "R4643", "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "original": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true }, { "record_id": "R3353", "field": "entity_id", "actual": "CYPRESS-REF-27033", "original": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "record_id": "R3353", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "original": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "record_id": "R3353", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R3353", "field": "original_principal_usd", "actual": "5880000", "original": "5880000", "expected": 5880000, "passed": true }, { "record_id": "R3353", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R3353", "field": "contact_name", "actual": "Sam Cypress4", "original": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "record_id": "R3353", "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "original": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true }, { "record_id": "R2253", "field": "entity_id", "actual": "CYPRESS-REF-14488", "original": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "record_id": "R2253", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "original": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "record_id": "R2253", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R2253", "field": "original_principal_usd", "actual": "6240000", "original": "6240000", "expected": 6240000, "passed": true }, { "record_id": "R2253", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R2253", "field": "contact_name", "actual": "Casey Cypress5", "original": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "record_id": "R2253", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R5546", "field": "entity_id", "actual": "CYPRESS-REF-65445", "original": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "record_id": "R5546", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "original": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "record_id": "R5546", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R5546", "field": "original_principal_usd", "actual": "6600000", "original": "6600000", "expected": 6600000, "passed": true }, { "record_id": "R5546", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R5546", "field": "contact_name", "actual": "Riley Cypress6", "original": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "record_id": "R5546", "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "original": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true }, { "record_id": "R7918", "field": "entity_id", "actual": "CYPRESS-REF-34203", "original": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "record_id": "R7918", "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "original": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "record_id": "R7918", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R7918", "field": "original_principal_usd", "actual": "6960000", "original": "6960000", "expected": 6960000, "passed": true }, { "record_id": "R7918", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-cypress/turn-01/tool-01.json" }, { "record_id": "R7918", "field": "contact_name", "actual": "Avery Cypress7", "original": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "record_id": "R7918", "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "original": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R2362", "passed": true, "expected": [ "S-R2362", "S-R7391" ], "actual": [ "S-R2362", "S-R7391" ] }, { "record_id": "R4643", "passed": true, "expected": [ "S-R4643" ], "actual": [ "S-R4643", "U-R4643-2" ] }, { "record_id": "R3353", "passed": true, "expected": [ "S-R3353" ], "actual": [ "S-R3353", "U-R3353-3" ] }, { "record_id": "R2253", "passed": true, "expected": [ "S-R2253" ], "actual": [ "S-R2253" ] }, { "record_id": "R5546", "passed": true, "expected": [ "S-R5546" ], "actual": [ "S-R5546" ] }, { "record_id": "R7918", "passed": true, "expected": [ "S-R7918" ], "actual": [ "S-R7918" ] } ], "source_sha256": "e1fdbefd562c4b99f422aa31bbb9f221c0c5fc325858106c976b3fc60e18de91" } }, { "case_id": "refinance-leads-cypress", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:37:26.583368+00:00", "completed_at": "2026-09-07T02:39:07.893680+00:00", "elapsed_seconds": 101.31, "cost_usd": 0.11019625, "known_partial_cost_usd": 0.11019625, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7061, "completion_tokens": 10137, "total_tokens": 17198, "cost": 0.11019625, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7168, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11019625, "gateway_cost": 0.11019625 }, "finish_reason": "stop", "elapsed_seconds": 101.297, "completed_at": "2026-09-07T02:39:07.892252+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/refinance-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-cypress", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 12, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R3917", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R7391", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 4800000, "expected": 4800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7391" ], "reported_sources": [ "S-R7391" ] }, { "record_id": "R4643", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 5520000, "expected": 5520000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4643" ], "reported_sources": [ "S-R4643" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353" ], "reported_sources": [ "S-R3353" ] }, { "record_id": "R3917", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-21679", "expected": "CYPRESS-REF-21679", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-21679", "expected": "O-CYPRESS-REF-21679", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress18", "expected": "Riley Cypress18", "passed": true }, { "field": "contact_email", "actual": "finance18@refinance-leads-cypress.example", "expected": "finance18@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3917" ], "reported_sources": [ "S-R3917" ] }, { "record_id": "R2253", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 6240000, "expected": 6240000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2253" ], "reported_sources": [ "S-R2253" ] }, { "record_id": "R5546", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 6600000, "expected": 6600000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5546" ], "reported_sources": [ "S-R5546" ] }, { "record_id": "R7918", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 6960000, "expected": 6960000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7918" ], "reported_sources": [ "S-R7918" ] } ], "decision_checks": [ { "record_id": "R3947", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R3917", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3389", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7918", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2362", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9569", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R9318", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9030", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8853", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2334", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R2570", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2253", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4643", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3709", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4862", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5546", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7391", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3836", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-cypress", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:37:39.601301+00:00", "completed_at": "2026-09-07T02:39:25.992576+00:00", "elapsed_seconds": 106.391, "cost_usd": 0.210265, "known_partial_cost_usd": 0.210265, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10108, "completion_tokens": 6389, "total_tokens": 16497, "cost": 0.210265, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1329, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.210265, "gateway_cost": 0.210265 }, "finish_reason": "stop", "elapsed_seconds": 106.379, "completed_at": "2026-09-07T02:39:25.990834+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/refinance-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-cypress", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 0.8333333333333334, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 5, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2362", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-62661", "expected": "CYPRESS-REF-62661", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-62661", "expected": "O-CYPRESS-REF-62661", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 4800000, "expected": 4800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress1", "expected": "Casey Cypress1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-cypress.example", "expected": "finance1@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2362" ], "reported_sources": [ "S-R2362", "S-R7391" ] }, { "record_id": "R4643", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-10959", "expected": "CYPRESS-REF-10959", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-10959", "expected": "O-CYPRESS-REF-10959", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 5520000, "expected": 5520000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress3", "expected": "Avery Cypress3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-cypress.example", "expected": "finance3@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4643" ], "reported_sources": [ "S-R4643" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-27033", "expected": "CYPRESS-REF-27033", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-27033", "expected": "O-CYPRESS-REF-27033", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 5880000, "expected": 5880000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Cypress4", "expected": "Sam Cypress4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-cypress.example", "expected": "finance4@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": false, "required_sources": [ "S-R3353" ], "reported_sources": [ "S-R3353", "U-R3353-3" ] }, { "record_id": "R2253", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-14488", "expected": "CYPRESS-REF-14488", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-14488", "expected": "O-CYPRESS-REF-14488", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 6240000, "expected": 6240000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Cypress5", "expected": "Casey Cypress5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2253" ], "reported_sources": [ "S-R2253" ] }, { "record_id": "R5546", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-65445", "expected": "CYPRESS-REF-65445", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-65445", "expected": "O-CYPRESS-REF-65445", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 6600000, "expected": 6600000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Cypress6", "expected": "Riley Cypress6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-cypress.example", "expected": "finance6@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5546" ], "reported_sources": [ "S-R5546" ] }, { "record_id": "R7918", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REF-34203", "expected": "CYPRESS-REF-34203", "passed": true }, { "field": "owner_entity_id", "actual": "O-CYPRESS-REF-34203", "expected": "O-CYPRESS-REF-34203", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 6960000, "expected": 6960000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Cypress7", "expected": "Avery Cypress7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-cypress.example", "expected": "finance7@refinance-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7918" ], "reported_sources": [ "S-R7918" ] } ], "decision_checks": [ { "record_id": "R3947", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R3917", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3389", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7918", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2362", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9569", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R9318", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R3320", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9030", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8853", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2334", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R2570", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2253", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4643", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3709", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4862", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5546", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7391", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3836", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-dogwood", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:38:07.709Z", "observed_completed_at": "2026-09-07T02:44:54.411Z", "packet_sha256": "12eab984dce08be697c45f8971816008bf1d3376e7457a69dde2af409b2ec417", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/dogwood_refinance_prospects.xlsx", "sha256": "9de23b2c18d2a3b5dd77eb3712a4ded8c075847fafb7562e24c649f595c1b9f6", "bytes": 7641 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.6224685, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 140.07, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "35e41bfe-d988-4024-b67d-cc607635c678", "session_id": "29433ebb-652d-417e-8a9a-f8b1b83fcb38", "started_at": "2026-09-06T22:39:51.981000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.4305385, 0.19193 ], "total_cost_usd": 0.6224685, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 140.07, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/refinance-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-dogwood", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3529", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 16800000, "expected": 16800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3529" ], "reported_sources": [ "S-R3529" ] }, { "record_id": "R9410", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 19320000, "expected": 19320000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9410" ], "reported_sources": [ "S-R9410" ] }, { "record_id": "R8389", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 20580000, "expected": 20580000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8389" ], "reported_sources": [ "S-R8389" ] }, { "record_id": "R2818", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 21840000, "expected": 21840000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2818" ], "reported_sources": [ "S-R2818" ] }, { "record_id": "R1318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 23100000, "expected": 23100000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1318" ], "reported_sources": [ "S-R1318" ] }, { "record_id": "R5792", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 24360000, "expected": 24360000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5792" ], "reported_sources": [ "S-R5792" ] } ], "decision_checks": [ { "record_id": "R2320", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R3529", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5792", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6805", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R8788", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R8811", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7731", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5565", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4836", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R2818", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9410", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2226", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R1271", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8389", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4312", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R8536", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R1318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2326", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6522", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2110", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": false }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/dogwood_refinance_prospects.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "Requested .csv; delivered .xlsx" ], "sheets": [ "Refinance Prospects", "Candidate Decisions" ], "table_sheet": "Refinance Prospects", "original_record_order": [ "R3529", "R9410", "R8389", "R2818", "R1318", "R5792" ], "answer_record_order": [ "R3529", "R9410", "R8389", "R2818", "R1318", "R5792" ], "field_checks": [ { "record_id": "R3529", "field": "entity_id", "actual": "DOGWOOD-REF-43781", "original": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "record_id": "R3529", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "original": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "record_id": "R3529", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R3529", "field": "original_principal_usd", "actual": 16800000, "original": 16800000, "expected": 16800000, "passed": true }, { "record_id": "R3529", "field": "outstanding_balance_usd", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R3529", "field": "contact_name", "actual": "Casey Dogwood1", "original": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "record_id": "R3529", "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "original": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R9410", "field": "entity_id", "actual": "DOGWOOD-REF-92578", "original": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "record_id": "R9410", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "original": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "record_id": "R9410", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R9410", "field": "original_principal_usd", "actual": 19320000, "original": 19320000, "expected": 19320000, "passed": true }, { "record_id": "R9410", "field": "outstanding_balance_usd", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R9410", "field": "contact_name", "actual": "Avery Dogwood3", "original": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "record_id": "R9410", "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "original": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R8389", "field": "entity_id", "actual": "DOGWOOD-REF-32756", "original": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "record_id": "R8389", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "original": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "record_id": "R8389", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R8389", "field": "original_principal_usd", "actual": 20580000, "original": 20580000, "expected": 20580000, "passed": true }, { "record_id": "R8389", "field": "outstanding_balance_usd", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R8389", "field": "contact_name", "actual": "Sam Dogwood4", "original": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "record_id": "R8389", "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "original": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R2818", "field": "entity_id", "actual": "DOGWOOD-REF-21989", "original": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "record_id": "R2818", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "original": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "record_id": "R2818", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R2818", "field": "original_principal_usd", "actual": 21840000, "original": 21840000, "expected": 21840000, "passed": true }, { "record_id": "R2818", "field": "outstanding_balance_usd", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R2818", "field": "contact_name", "actual": "Casey Dogwood5", "original": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "record_id": "R2818", "field": "contact_email", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R1318", "field": "entity_id", "actual": "DOGWOOD-REF-14595", "original": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "record_id": "R1318", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "original": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "record_id": "R1318", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R1318", "field": "original_principal_usd", "actual": 23100000, "original": 23100000, "expected": 23100000, "passed": true }, { "record_id": "R1318", "field": "outstanding_balance_usd", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R1318", "field": "contact_name", "actual": "Riley Dogwood6", "original": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "record_id": "R1318", "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "original": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R5792", "field": "entity_id", "actual": "DOGWOOD-REF-69815", "original": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "record_id": "R5792", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "original": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "record_id": "R5792", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R5792", "field": "original_principal_usd", "actual": 24360000, "original": 24360000, "expected": 24360000, "passed": true }, { "record_id": "R5792", "field": "outstanding_balance_usd", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R5792", "field": "contact_name", "actual": "Avery Dogwood7", "original": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "record_id": "R5792", "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "original": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R3529", "passed": true, "expected": [ "S-R3529" ], "actual": [ "S-R3529" ] }, { "record_id": "R9410", "passed": true, "expected": [ "S-R9410" ], "actual": [ "S-R9410" ] }, { "record_id": "R8389", "passed": true, "expected": [ "S-R8389" ], "actual": [ "S-R8389" ] }, { "record_id": "R2818", "passed": true, "expected": [ "S-R2818" ], "actual": [ "S-R2818" ] }, { "record_id": "R1318", "passed": true, "expected": [ "S-R1318" ], "actual": [ "S-R1318" ] }, { "record_id": "R5792", "passed": true, "expected": [ "S-R5792" ], "actual": [ "S-R5792" ] } ], "source_sha256": "9de23b2c18d2a3b5dd77eb3712a4ded8c075847fafb7562e24c649f595c1b9f6" } }, { "case_id": "refinance-leads-dogwood", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:38:15.425255+00:00", "completed_at": "2026-09-07T02:43:08.187990+00:00", "elapsed_seconds": 292.762, "cost_usd": 0.29875799999999997, "known_partial_cost_usd": 0.29875799999999997, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 4, "tool_calls": 4, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7843, "completion_tokens": 5963, "total_tokens": 13806, "cost": 0.06943375, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5504, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06943375, "gateway_cost": 0.06943375 }, "finish_reason": "tool_calls", "elapsed_seconds": 55.993, "completed_at": "2026-09-07T02:39:11.429253+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8323, "completion_tokens": 6216, "total_tokens": 14539, "cost": 0.06377975, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7808, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5760, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06377975, "gateway_cost": 0.06377975 }, "finish_reason": "tool_calls", "elapsed_seconds": 71.538, "completed_at": "2026-09-07T02:40:22.973193+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8744, "completion_tokens": 7413, "total_tokens": 16157, "cost": 0.075844, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8192, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6976, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.075844, "gateway_cost": 0.075844 }, "finish_reason": "tool_calls", "elapsed_seconds": 76.349, "completed_at": "2026-09-07T02:41:39.325270+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9170, "completion_tokens": 8803, "total_tokens": 17973, "cost": 0.0897005, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8704, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6656, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0897005, "gateway_cost": 0.0897005 }, "finish_reason": "tool_calls", "elapsed_seconds": 88.857, "completed_at": "2026-09-07T02:43:08.185681+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "bfe6edd7e04c38048187d433ce0a19b47942ca2b3d140c657e56c74daaa4bf6d", "bytes": 1006 }, { "path": "artifacts/prospects-v2.csv", "sha256": "fb35f6acc2120060a57515c382b8e401db333c2e5029e18ff0c3285b9046059a", "bytes": 874 }, { "path": "artifacts/prospects-v3.csv", "sha256": "e4cd090c788b900a56ca60d0ed39e171f63ec6df986bfa61c05dd29877286b18", "bytes": 882 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-dogwood", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3529", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 16800000, "expected": 16800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3529" ], "reported_sources": [ "S-R3529", "S-R5565" ] }, { "record_id": "R9410", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 19320000, "expected": 19320000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9410" ], "reported_sources": [ "S-R9410" ] }, { "record_id": "R8389", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 20580000, "expected": 20580000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8389" ], "reported_sources": [ "S-R8389" ] }, { "record_id": "R2818", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 21840000, "expected": 21840000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2818" ], "reported_sources": [ "S-R2818" ] }, { "record_id": "R1318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 23100000, "expected": 23100000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1318" ], "reported_sources": [ "S-R1318" ] }, { "record_id": "R5792", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 24360000, "expected": 24360000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5792" ], "reported_sources": [ "S-R5792" ] } ], "decision_checks": [ { "record_id": "R2320", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R3529", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5792", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6805", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R8788", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R8811", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7731", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5565", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4836", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R2818", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9410", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2226", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R1271", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8389", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4312", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R8536", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R1318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2326", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6522", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2110", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v3.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R3529 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9410 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8389 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2818 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2818 / contact_email: artifact differs from answer or lacks explicit unknown", "R1318 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R5792 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R3529", "R9410", "R8389", "R2818", "R1318", "R5792" ], "answer_record_order": [ "R3529", "R9410", "R8389", "R2818", "R1318", "R5792" ], "field_checks": [ { "record_id": "R3529", "field": "entity_id", "actual": "DOGWOOD-REF-43781", "original": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "record_id": "R3529", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "original": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "record_id": "R3529", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R3529", "field": "original_principal_usd", "actual": "16800000", "original": "16800000", "expected": 16800000, "passed": true }, { "record_id": "R3529", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R3529", "field": "contact_name", "actual": "Casey Dogwood1", "original": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "record_id": "R3529", "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "original": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R9410", "field": "entity_id", "actual": "DOGWOOD-REF-92578", "original": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "record_id": "R9410", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "original": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "record_id": "R9410", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R9410", "field": "original_principal_usd", "actual": "19320000", "original": "19320000", "expected": 19320000, "passed": true }, { "record_id": "R9410", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R9410", "field": "contact_name", "actual": "Avery Dogwood3", "original": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "record_id": "R9410", "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "original": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R8389", "field": "entity_id", "actual": "DOGWOOD-REF-32756", "original": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "record_id": "R8389", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "original": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "record_id": "R8389", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R8389", "field": "original_principal_usd", "actual": "20580000", "original": "20580000", "expected": 20580000, "passed": true }, { "record_id": "R8389", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R8389", "field": "contact_name", "actual": "Sam Dogwood4", "original": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "record_id": "R8389", "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "original": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R2818", "field": "entity_id", "actual": "DOGWOOD-REF-21989", "original": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "record_id": "R2818", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "original": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "record_id": "R2818", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R2818", "field": "original_principal_usd", "actual": "21840000", "original": "21840000", "expected": 21840000, "passed": true }, { "record_id": "R2818", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R2818", "field": "contact_name", "actual": "Casey Dogwood5", "original": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "record_id": "R2818", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R1318", "field": "entity_id", "actual": "DOGWOOD-REF-14595", "original": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "record_id": "R1318", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "original": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "record_id": "R1318", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R1318", "field": "original_principal_usd", "actual": "23100000", "original": "23100000", "expected": 23100000, "passed": true }, { "record_id": "R1318", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R1318", "field": "contact_name", "actual": "Riley Dogwood6", "original": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "record_id": "R1318", "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "original": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R5792", "field": "entity_id", "actual": "DOGWOOD-REF-69815", "original": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "record_id": "R5792", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "original": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "record_id": "R5792", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R5792", "field": "original_principal_usd", "actual": "24360000", "original": "24360000", "expected": 24360000, "passed": true }, { "record_id": "R5792", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-dogwood/turn-03/tool-01.json" }, { "record_id": "R5792", "field": "contact_name", "actual": "Avery Dogwood7", "original": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "record_id": "R5792", "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "original": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R3529", "passed": true, "expected": [ "S-R3529", "S-R5565" ], "actual": [ "S-R3529", "S-R5565" ] }, { "record_id": "R9410", "passed": true, "expected": [ "S-R9410" ], "actual": [ "S-R9410" ] }, { "record_id": "R8389", "passed": true, "expected": [ "S-R8389" ], "actual": [ "S-R8389" ] }, { "record_id": "R2818", "passed": true, "expected": [ "S-R2818" ], "actual": [ "S-R2818" ] }, { "record_id": "R1318", "passed": true, "expected": [ "S-R1318" ], "actual": [ "S-R1318" ] }, { "record_id": "R5792", "passed": true, "expected": [ "S-R5792" ], "actual": [ "S-R5792" ] } ], "source_sha256": "e4cd090c788b900a56ca60d0ed39e171f63ec6df986bfa61c05dd29877286b18" } }, { "case_id": "refinance-leads-dogwood", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:39:07.894626+00:00", "completed_at": "2026-09-07T02:40:07.308925+00:00", "elapsed_seconds": 59.414, "cost_usd": 0.28210999999999997, "known_partial_cost_usd": 0.28210999999999997, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 1, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12530, "completion_tokens": 1704, "total_tokens": 14234, "cost": 0.10525, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 931, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10525, "gateway_cost": 0.10525 }, "finish_reason": "tool_calls", "elapsed_seconds": 17.776, "completed_at": "2026-09-07T02:39:25.688353+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13307, "completion_tokens": 4413, "total_tokens": 17720, "cost": 0.17686, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 486, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.17686, "gateway_cost": 0.17686 }, "finish_reason": "stop", "elapsed_seconds": 41.616, "completed_at": "2026-09-07T02:40:07.307278+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "eb4434d1280811929cb68930a83922535068b02bbf01fed7cf73814aa372973a", "bytes": 1012 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-dogwood", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3529", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 16800000, "expected": 16800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3529" ], "reported_sources": [ "S-R3529", "S-R5565" ] }, { "record_id": "R9410", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 19320000, "expected": 19320000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9410" ], "reported_sources": [ "S-R9410" ] }, { "record_id": "R8389", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 20580000, "expected": 20580000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8389" ], "reported_sources": [ "S-R8389" ] }, { "record_id": "R2818", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 21840000, "expected": 21840000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2818" ], "reported_sources": [ "S-R2818" ] }, { "record_id": "R1318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 23100000, "expected": 23100000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1318" ], "reported_sources": [ "S-R1318" ] }, { "record_id": "R5792", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 24360000, "expected": 24360000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5792" ], "reported_sources": [ "S-R5792" ] } ], "decision_checks": [ { "record_id": "R2320", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R3529", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5792", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6805", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R8788", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R8811", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7731", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5565", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4836", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R2818", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9410", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2226", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R1271", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8389", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4312", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R8536", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R1318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2326", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6522", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2110", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "7 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R3529 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R9410 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R8389 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2818 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R2818 / contact_email: artifact differs from answer or lacks explicit unknown", "R1318 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R5792 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R3529", "R9410", "R8389", "R2818", "R1318", "R5792" ], "answer_record_order": [ "R3529", "R9410", "R8389", "R2818", "R1318", "R5792" ], "field_checks": [ { "record_id": "R3529", "field": "entity_id", "actual": "DOGWOOD-REF-43781", "original": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "record_id": "R3529", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "original": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "record_id": "R3529", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R3529", "field": "original_principal_usd", "actual": "16800000", "original": "16800000", "expected": 16800000, "passed": true }, { "record_id": "R3529", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R3529", "field": "contact_name", "actual": "Casey Dogwood1", "original": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "record_id": "R3529", "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "original": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R9410", "field": "entity_id", "actual": "DOGWOOD-REF-92578", "original": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "record_id": "R9410", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "original": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "record_id": "R9410", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R9410", "field": "original_principal_usd", "actual": "19320000", "original": "19320000", "expected": 19320000, "passed": true }, { "record_id": "R9410", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R9410", "field": "contact_name", "actual": "Avery Dogwood3", "original": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "record_id": "R9410", "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "original": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R8389", "field": "entity_id", "actual": "DOGWOOD-REF-32756", "original": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "record_id": "R8389", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "original": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "record_id": "R8389", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R8389", "field": "original_principal_usd", "actual": "20580000", "original": "20580000", "expected": 20580000, "passed": true }, { "record_id": "R8389", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R8389", "field": "contact_name", "actual": "Sam Dogwood4", "original": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "record_id": "R8389", "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "original": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R2818", "field": "entity_id", "actual": "DOGWOOD-REF-21989", "original": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "record_id": "R2818", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "original": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "record_id": "R2818", "field": "maturity_date", "actual": "2027-05-15", "original": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "record_id": "R2818", "field": "original_principal_usd", "actual": "21840000", "original": "21840000", "expected": 21840000, "passed": true }, { "record_id": "R2818", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R2818", "field": "contact_name", "actual": "Casey Dogwood5", "original": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "record_id": "R2818", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R1318", "field": "entity_id", "actual": "DOGWOOD-REF-14595", "original": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "record_id": "R1318", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "original": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "record_id": "R1318", "field": "maturity_date", "actual": "2027-06-15", "original": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "record_id": "R1318", "field": "original_principal_usd", "actual": "23100000", "original": "23100000", "expected": 23100000, "passed": true }, { "record_id": "R1318", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R1318", "field": "contact_name", "actual": "Riley Dogwood6", "original": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "record_id": "R1318", "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "original": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true }, { "record_id": "R5792", "field": "entity_id", "actual": "DOGWOOD-REF-69815", "original": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "record_id": "R5792", "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "original": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "record_id": "R5792", "field": "maturity_date", "actual": "2027-07-15", "original": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "record_id": "R5792", "field": "original_principal_usd", "actual": "24360000", "original": "24360000", "expected": 24360000, "passed": true }, { "record_id": "R5792", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R5792", "field": "contact_name", "actual": "Avery Dogwood7", "original": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "record_id": "R5792", "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "original": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "harness_induced_null_annotations": 7, "source_checks": [ { "record_id": "R3529", "passed": true, "expected": [ "S-R3529", "S-R5565" ], "actual": [ "S-R3529", "S-R5565" ] }, { "record_id": "R9410", "passed": true, "expected": [ "S-R9410" ], "actual": [ "S-R9410", "U-R9410-2" ] }, { "record_id": "R8389", "passed": true, "expected": [ "S-R8389" ], "actual": [ "S-R8389", "U-R8389-3" ] }, { "record_id": "R2818", "passed": true, "expected": [ "S-R2818" ], "actual": [ "S-R2818" ] }, { "record_id": "R1318", "passed": true, "expected": [ "S-R1318" ], "actual": [ "S-R1318" ] }, { "record_id": "R5792", "passed": true, "expected": [ "S-R5792" ], "actual": [ "S-R5792" ] } ], "source_sha256": "eb4434d1280811929cb68930a83922535068b02bbf01fed7cf73814aa372973a" } }, { "case_id": "refinance-leads-dogwood", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:39:25.993463+00:00", "completed_at": "2026-09-07T02:41:49.193391+00:00", "elapsed_seconds": 143.199, "cost_usd": 0.10348125, "known_partial_cost_usd": 0.10348125, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7097, "completion_tokens": 9461, "total_tokens": 16558, "cost": 0.10348125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6976, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10348125, "gateway_cost": 0.10348125 }, "finish_reason": "stop", "elapsed_seconds": 143.18, "completed_at": "2026-09-07T02:41:49.191152+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/refinance-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-dogwood", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3529", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 16800000, "expected": 16800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3529" ], "reported_sources": [ "S-R3529", "S-R5565" ] }, { "record_id": "R9410", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 19320000, "expected": 19320000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9410" ], "reported_sources": [ "S-R9410" ] }, { "record_id": "R8389", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 20580000, "expected": 20580000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8389" ], "reported_sources": [ "S-R8389" ] }, { "record_id": "R2818", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 21840000, "expected": 21840000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2818" ], "reported_sources": [ "S-R2818" ] }, { "record_id": "R1318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 23100000, "expected": 23100000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1318" ], "reported_sources": [ "S-R1318" ] }, { "record_id": "R5792", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 24360000, "expected": 24360000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5792" ], "reported_sources": [ "S-R5792" ] } ], "decision_checks": [ { "record_id": "R2320", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R3529", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5792", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6805", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R8788", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R8811", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7731", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5565", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4836", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R2818", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9410", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2226", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R1271", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8389", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4312", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R8536", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R1318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2326", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6522", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2110", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-dogwood", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:40:07.309787+00:00", "completed_at": "2026-09-07T02:41:08.332238+00:00", "elapsed_seconds": 61.022, "cost_usd": 0.2075, "known_partial_cost_usd": 0.2075, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10245, "completion_tokens": 6251, "total_tokens": 16496, "cost": 0.2075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1174, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.2075, "gateway_cost": 0.2075 }, "finish_reason": "stop", "elapsed_seconds": 61.008, "completed_at": "2026-09-07T02:41:08.330648+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/refinance-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-dogwood", "task": "refinance_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 7, "total": 7 }, "ineligible_rejection": { "passed": 13, "total": 13 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3529", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-43781", "expected": "DOGWOOD-REF-43781", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-43781", "expected": "O-DOGWOOD-REF-43781", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 16800000, "expected": 16800000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood1", "expected": "Casey Dogwood1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-dogwood.example", "expected": "finance1@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3529" ], "reported_sources": [ "S-R3529", "S-R5565" ] }, { "record_id": "R9410", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-92578", "expected": "DOGWOOD-REF-92578", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-92578", "expected": "O-DOGWOOD-REF-92578", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 19320000, "expected": 19320000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood3", "expected": "Avery Dogwood3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-dogwood.example", "expected": "finance3@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9410" ], "reported_sources": [ "S-R9410" ] }, { "record_id": "R8389", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-32756", "expected": "DOGWOOD-REF-32756", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-32756", "expected": "O-DOGWOOD-REF-32756", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 20580000, "expected": 20580000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Sam Dogwood4", "expected": "Sam Dogwood4", "passed": true }, { "field": "contact_email", "actual": "finance4@refinance-leads-dogwood.example", "expected": "finance4@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8389" ], "reported_sources": [ "S-R8389" ] }, { "record_id": "R2818", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-21989", "expected": "DOGWOOD-REF-21989", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-21989", "expected": "O-DOGWOOD-REF-21989", "passed": true }, { "field": "maturity_date", "actual": "2027-05-15", "expected": "2027-05-15", "passed": true }, { "field": "original_principal_usd", "actual": 21840000, "expected": 21840000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Dogwood5", "expected": "Casey Dogwood5", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2818" ], "reported_sources": [ "S-R2818" ] }, { "record_id": "R1318", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-14595", "expected": "DOGWOOD-REF-14595", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-14595", "expected": "O-DOGWOOD-REF-14595", "passed": true }, { "field": "maturity_date", "actual": "2027-06-15", "expected": "2027-06-15", "passed": true }, { "field": "original_principal_usd", "actual": 23100000, "expected": 23100000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Dogwood6", "expected": "Riley Dogwood6", "passed": true }, { "field": "contact_email", "actual": "finance6@refinance-leads-dogwood.example", "expected": "finance6@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1318" ], "reported_sources": [ "S-R1318" ] }, { "record_id": "R5792", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REF-69815", "expected": "DOGWOOD-REF-69815", "passed": true }, { "field": "owner_entity_id", "actual": "O-DOGWOOD-REF-69815", "expected": "O-DOGWOOD-REF-69815", "passed": true }, { "field": "maturity_date", "actual": "2027-07-15", "expected": "2027-07-15", "passed": true }, { "field": "original_principal_usd", "actual": 24360000, "expected": 24360000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Dogwood7", "expected": "Avery Dogwood7", "passed": true }, { "field": "contact_email", "actual": "finance7@refinance-leads-dogwood.example", "expected": "finance7@refinance-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5792" ], "reported_sources": [ "S-R5792" ] } ], "decision_checks": [ { "record_id": "R2320", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R3529", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5792", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6805", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R8788", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R8811", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7731", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5565", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4836", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R2818", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9410", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2226", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R1271", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8389", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4312", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R8536", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R1318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2326", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6522", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2110", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-elm", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:41:11.963Z", "observed_completed_at": "2026-09-07T02:44:55.549Z", "packet_sha256": "91f4b7a78be924e50e9b3a57cd65e53f1f4fa873e0f15d2fed45e93e4a5368d8", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/refinance_leads_elm_prospects.csv", "sha256": "95e67df40e298d4f9a64d02f059e1f5c5190d8ce816c2fc3e521e2ac2693ca77", "bytes": 533 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.5288765, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 111.15, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "047d27bd-7b7b-4266-af7d-f2eb1998b999", "session_id": "365b5fef-5418-406a-8ce0-86cce0d5a883", "started_at": "2026-09-06T22:42:34.276000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.38167475, 0.14720175 ], "total_cost_usd": 0.5288765, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 111.15, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/refinance-leads-elm", "expected_target": 2, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-elm", "task": "refinance_leads", "target_count": 2, "available_unique": 2, "returned_count": 3, "eligible_count": 2, "verified_count": 2, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.6666666666666666, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 3, "total": 3 }, "ineligible_rejection": { "passed": 16, "total": 17 }, "ranking_pair_agreement": { "passed": 1, "total": 1 }, "material_errors": [ { "record_id": "R7798", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 8640000, "expected": 8640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968" ] }, { "record_id": "R7631", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 9936000, "expected": 9936000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7631" ], "reported_sources": [ "S-R7631" ] }, { "record_id": "R7798", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-31274", "expected": "ELM-REF-31274", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-31274", "expected": "O-ELM-REF-31274", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 10584000, "expected": 10584000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Elm18", "expected": "Riley Elm18", "passed": true }, { "field": "contact_email", "actual": "finance18@refinance-leads-elm.example", "expected": "finance18@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7798" ], "reported_sources": [ "S-R7798" ] } ], "decision_checks": [ { "record_id": "R7798", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1548", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5893", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6909", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2801", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R7631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4392", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4755", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3427", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4641", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R3478", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9095", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9201", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2376", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1927", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R9233", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8058", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2630", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4437", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/refinance_leads_elm_prospects.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "R6968 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7631 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7798 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R6968", "R7631", "R7798" ], "answer_record_order": [ "R6968", "R7631", "R7798" ], "field_checks": [ { "record_id": "R6968", "field": "entity_id", "actual": "ELM-REF-63521", "original": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "record_id": "R6968", "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "original": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "record_id": "R6968", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R6968", "field": "original_principal_usd", "actual": "8640000", "original": "8640000", "expected": 8640000, "passed": true }, { "record_id": "R6968", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R6968", "field": "contact_name", "actual": "Casey Elm1", "original": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "record_id": "R6968", "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "original": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true }, { "record_id": "R7631", "field": "entity_id", "actual": "ELM-REF-53828", "original": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "record_id": "R7631", "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "original": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "record_id": "R7631", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R7631", "field": "original_principal_usd", "actual": "9936000", "original": "9936000", "expected": 9936000, "passed": true }, { "record_id": "R7631", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7631", "field": "contact_name", "actual": "Avery Elm3", "original": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "record_id": "R7631", "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "original": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true }, { "record_id": "R7798", "field": "entity_id", "actual": "ELM-REF-31274", "original": "ELM-REF-31274", "expected": "ELM-REF-31274", "passed": true }, { "record_id": "R7798", "field": "owner_entity_id", "actual": "O-ELM-REF-31274", "original": "O-ELM-REF-31274", "expected": "O-ELM-REF-31274", "passed": true }, { "record_id": "R7798", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R7798", "field": "original_principal_usd", "actual": "10584000", "original": "10584000", "expected": 10584000, "passed": true }, { "record_id": "R7798", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R7798", "field": "contact_name", "actual": "Riley Elm18", "original": "Riley Elm18", "expected": "Riley Elm18", "passed": true }, { "record_id": "R7798", "field": "contact_email", "actual": "finance18@refinance-leads-elm.example", "original": "finance18@refinance-leads-elm.example", "expected": "finance18@refinance-leads-elm.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R6968", "passed": true, "expected": [ "S-R6968" ], "actual": [ "S-R6968" ] }, { "record_id": "R7631", "passed": true, "expected": [ "S-R7631" ], "actual": [ "S-R7631" ] }, { "record_id": "R7798", "passed": true, "expected": [ "S-R7798" ], "actual": [ "S-R7798" ] } ], "source_sha256": "95e67df40e298d4f9a64d02f059e1f5c5190d8ce816c2fc3e521e2ac2693ca77" } }, { "case_id": "refinance-leads-elm", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:41:08.333102+00:00", "completed_at": "2026-09-07T02:47:27.729377+00:00", "elapsed_seconds": 379.395, "cost_usd": 0.3128265, "known_partial_cost_usd": 0.3128265, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 5, "tool_calls": 5, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8064, "completion_tokens": 5299, "total_tokens": 13363, "cost": 0.06307, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5056, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06307, "gateway_cost": 0.06307 }, "finish_reason": "tool_calls", "elapsed_seconds": 68.612, "completed_at": "2026-09-07T02:42:16.957808+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8317, "completion_tokens": 6084, "total_tokens": 14401, "cost": 0.06230825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7936, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5888, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06230825, "gateway_cost": 0.06230825 }, "finish_reason": "tool_calls", "elapsed_seconds": 85.158, "completed_at": "2026-09-07T02:43:42.119914+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8512, "completion_tokens": 5383, "total_tokens": 13895, "cost": 0.055254, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8192, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5184, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.055254, "gateway_cost": 0.055254 }, "finish_reason": "tool_calls", "elapsed_seconds": 68.699, "completed_at": "2026-09-07T02:44:50.822270+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8707, "completion_tokens": 5921, "total_tokens": 14628, "cost": 0.06058975, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8448, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5632, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06058975, "gateway_cost": 0.06058975 }, "finish_reason": "tool_calls", "elapsed_seconds": 58.096, "completed_at": "2026-09-07T02:45:48.921382+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8962, "completion_tokens": 7005, "total_tokens": 15967, "cost": 0.0716045, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8576, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5376, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0716045, "gateway_cost": 0.0716045 }, "finish_reason": "tool_calls", "elapsed_seconds": 98.802, "completed_at": "2026-09-07T02:47:27.725314+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "b5399e7694e90ff2c4cdfbf548dccd98183fa487f68e83efb96808c8b2a34c1f", "bytes": 490 }, { "path": "artifacts/prospects-v2.csv", "sha256": "2f232017e3f325e7a6502f30726f3a0e85b222512dfd5a50a3697247da833925", "bytes": 370 }, { "path": "artifacts/prospects-v3.csv", "sha256": "2f232017e3f325e7a6502f30726f3a0e85b222512dfd5a50a3697247da833925", "bytes": 370 }, { "path": "artifacts/prospects-v4.csv", "sha256": "cbcdd5797a848f8ca3589e6a2396888859c26b2332f0f27a734ba315e63b50de", "bytes": 486 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-elm", "expected_target": 2, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-elm", "task": "refinance_leads", "target_count": 2, "available_unique": 2, "returned_count": 3, "eligible_count": 2, "verified_count": 2, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.6666666666666666, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 3, "total": 3 }, "ineligible_rejection": { "passed": 16, "total": 17 }, "ranking_pair_agreement": { "passed": 1, "total": 1 }, "material_errors": [ { "record_id": "R7798", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 8640000, "expected": 8640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968", "S-R4437" ] }, { "record_id": "R7631", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 9936000, "expected": 9936000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7631" ], "reported_sources": [ "S-R7631" ] }, { "record_id": "R7798", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-31274", "expected": "ELM-REF-31274", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-31274", "expected": "O-ELM-REF-31274", "passed": true }, { "field": "maturity_date", "actual": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "field": "original_principal_usd", "actual": 10584000, "expected": 10584000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Riley Elm18", "expected": "Riley Elm18", "passed": true }, { "field": "contact_email", "actual": "finance18@refinance-leads-elm.example", "expected": "finance18@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7798" ], "reported_sources": [ "S-R7798" ] } ], "decision_checks": [ { "record_id": "R7798", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1548", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5893", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6909", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2801", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R7631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4392", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4755", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3427", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4641", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R3478", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9095", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9201", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2376", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1927", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R9233", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8058", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2630", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4437", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v4.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "3 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R6968 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7631 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7798 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R6968", "R7631", "R7798" ], "answer_record_order": [ "R6968", "R7631", "R7798" ], "field_checks": [ { "record_id": "R6968", "field": "entity_id", "actual": "ELM-REF-63521", "original": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "record_id": "R6968", "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "original": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "record_id": "R6968", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R6968", "field": "original_principal_usd", "actual": "8640000", "original": "8640000", "expected": 8640000, "passed": true }, { "record_id": "R6968", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-elm/turn-04/tool-01.json" }, { "record_id": "R6968", "field": "contact_name", "actual": "Casey Elm1", "original": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "record_id": "R6968", "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "original": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true }, { "record_id": "R7631", "field": "entity_id", "actual": "ELM-REF-53828", "original": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "record_id": "R7631", "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "original": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "record_id": "R7631", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R7631", "field": "original_principal_usd", "actual": "9936000", "original": "9936000", "expected": 9936000, "passed": true }, { "record_id": "R7631", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-elm/turn-04/tool-01.json" }, { "record_id": "R7631", "field": "contact_name", "actual": "Avery Elm3", "original": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "record_id": "R7631", "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "original": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true }, { "record_id": "R7798", "field": "entity_id", "actual": "ELM-REF-31274", "original": "ELM-REF-31274", "expected": "ELM-REF-31274", "passed": true }, { "record_id": "R7798", "field": "owner_entity_id", "actual": "O-ELM-REF-31274", "original": "O-ELM-REF-31274", "expected": "O-ELM-REF-31274", "passed": true }, { "record_id": "R7798", "field": "maturity_date", "actual": "2027-04-15", "original": "2027-04-15", "expected": "2027-04-15", "passed": true }, { "record_id": "R7798", "field": "original_principal_usd", "actual": "10584000", "original": "10584000", "expected": 10584000, "passed": true }, { "record_id": "R7798", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-elm/turn-04/tool-01.json" }, { "record_id": "R7798", "field": "contact_name", "actual": "Riley Elm18", "original": "Riley Elm18", "expected": "Riley Elm18", "passed": true }, { "record_id": "R7798", "field": "contact_email", "actual": "finance18@refinance-leads-elm.example", "original": "finance18@refinance-leads-elm.example", "expected": "finance18@refinance-leads-elm.example", "passed": true } ], "harness_induced_null_annotations": 3, "source_checks": [ { "record_id": "R6968", "passed": true, "expected": [ "S-R4437", "S-R6968" ], "actual": [ "S-R4437", "S-R6968" ] }, { "record_id": "R7631", "passed": true, "expected": [ "S-R7631" ], "actual": [ "S-R7631" ] }, { "record_id": "R7798", "passed": true, "expected": [ "S-R7798" ], "actual": [ "S-R7798" ] } ], "source_sha256": "cbcdd5797a848f8ca3589e6a2396888859c26b2332f0f27a734ba315e63b50de" } }, { "case_id": "refinance-leads-elm", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:41:49.194196+00:00", "completed_at": "2026-09-07T02:42:57.454755+00:00", "elapsed_seconds": 68.26, "cost_usd": 0.28809999999999997, "known_partial_cost_usd": 0.28809999999999997, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12536, "completion_tokens": 1524, "total_tokens": 14060, "cost": 0.10078, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1128, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10078, "gateway_cost": 0.10078 }, "finish_reason": "tool_calls", "elapsed_seconds": 16.557, "completed_at": "2026-09-07T02:42:05.763842+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12994, "completion_tokens": 4894, "total_tokens": 17888, "cost": 0.18732, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1044, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.18732, "gateway_cost": 0.18732 }, "finish_reason": "tool_calls", "elapsed_seconds": 51.683, "completed_at": "2026-09-07T02:42:57.451435+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "138757cbfb49255d7a34308889f45443bd50cdc86dcb8c5aa95fd4b3b479c545", "bytes": 629 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-elm", "expected_target": 2, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-elm", "task": "refinance_leads", "target_count": 2, "available_unique": 2, "returned_count": 2, "eligible_count": 2, "verified_count": 2, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 14, "total": 14 }, "evidence_accuracy": { "passed": 2, "total": 2 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 3, "total": 3 }, "ineligible_rejection": { "passed": 17, "total": 17 }, "ranking_pair_agreement": { "passed": 1, "total": 1 }, "material_errors": [], "selected_checks": [ { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 8640000, "expected": 8640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968", "S-R4437" ] }, { "record_id": "R7631", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 9936000, "expected": 9936000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7631" ], "reported_sources": [ "S-R7631" ] } ], "decision_checks": [ { "record_id": "R7798", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1548", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5893", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6909", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2801", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R7631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4392", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4755", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3427", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4641", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R3478", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9095", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9201", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2376", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1927", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R9233", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8058", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2630", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4437", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "2 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R6968 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown", "R7631 / outstanding_balance_usd: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R6968", "R7631" ], "answer_record_order": [ "R6968", "R7631" ], "field_checks": [ { "record_id": "R6968", "field": "entity_id", "actual": "ELM-REF-63521", "original": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "record_id": "R6968", "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "original": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "record_id": "R6968", "field": "maturity_date", "actual": "2027-01-15", "original": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "record_id": "R6968", "field": "original_principal_usd", "actual": "8640000", "original": "8640000", "expected": 8640000, "passed": true }, { "record_id": "R6968", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-elm/turn-01/tool-01.json" }, { "record_id": "R6968", "field": "contact_name", "actual": "Casey Elm1", "original": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "record_id": "R6968", "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "original": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true }, { "record_id": "R7631", "field": "entity_id", "actual": "ELM-REF-53828", "original": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "record_id": "R7631", "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "original": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "record_id": "R7631", "field": "maturity_date", "actual": "2027-03-15", "original": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "record_id": "R7631", "field": "original_principal_usd", "actual": "9936000", "original": "9936000", "expected": 9936000, "passed": true }, { "record_id": "R7631", "field": "outstanding_balance_usd", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-elm/turn-01/tool-01.json" }, { "record_id": "R7631", "field": "contact_name", "actual": "Avery Elm3", "original": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "record_id": "R7631", "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "original": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true } ], "harness_induced_null_annotations": 2, "source_checks": [ { "record_id": "R6968", "passed": true, "expected": [ "S-R4437", "S-R6968" ], "actual": [ "S-R4437", "S-R6968" ] }, { "record_id": "R7631", "passed": true, "expected": [ "S-R7631" ], "actual": [ "S-R7631", "U-R7631-2" ] } ], "source_sha256": "138757cbfb49255d7a34308889f45443bd50cdc86dcb8c5aa95fd4b3b479c545" } }, { "case_id": "refinance-leads-elm", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:42:57.455684+00:00", "completed_at": "2026-09-07T02:44:56.104872+00:00", "elapsed_seconds": 118.649, "cost_usd": 0.0806675, "known_partial_cost_usd": 0.0806675, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7318, "completion_tokens": 7152, "total_tokens": 14470, "cost": 0.0806675, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5312, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0806675, "gateway_cost": 0.0806675 }, "finish_reason": "stop", "elapsed_seconds": 118.635, "completed_at": "2026-09-07T02:44:56.103123+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/refinance-leads-elm", "expected_target": 2, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-elm", "task": "refinance_leads", "target_count": 2, "available_unique": 2, "returned_count": 2, "eligible_count": 2, "verified_count": 2, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 14, "total": 14 }, "evidence_accuracy": { "passed": 2, "total": 2 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 3, "total": 3 }, "ineligible_rejection": { "passed": 17, "total": 17 }, "ranking_pair_agreement": { "passed": 1, "total": 1 }, "material_errors": [], "selected_checks": [ { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 8640000, "expected": 8640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968", "S-R4437" ] }, { "record_id": "R7631", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 9936000, "expected": 9936000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7631" ], "reported_sources": [ "S-R7631" ] } ], "decision_checks": [ { "record_id": "R7798", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1548", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5893", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6909", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2801", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R7631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4392", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4755", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3427", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4641", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R3478", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9095", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9201", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2376", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1927", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R9233", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8058", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2630", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4437", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-elm", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:43:08.188928+00:00", "completed_at": "2026-09-07T02:44:01.713260+00:00", "elapsed_seconds": 53.524, "cost_usd": 0.182455, "known_partial_cost_usd": 0.182455, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10251, "completion_tokens": 5248, "total_tokens": 15499, "cost": 0.182455, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1113, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.182455, "gateway_cost": 0.182455 }, "finish_reason": "stop", "elapsed_seconds": 53.511, "completed_at": "2026-09-07T02:44:01.711697+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/refinance-leads-elm", "expected_target": 2, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-elm", "task": "refinance_leads", "target_count": 2, "available_unique": 2, "returned_count": 2, "eligible_count": 2, "verified_count": 2, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 14, "total": 14 }, "evidence_accuracy": { "passed": 2, "total": 2 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 3, "total": 3 }, "ineligible_rejection": { "passed": 17, "total": 17 }, "ranking_pair_agreement": { "passed": 1, "total": 1 }, "material_errors": [], "selected_checks": [ { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-63521", "expected": "ELM-REF-63521", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-63521", "expected": "O-ELM-REF-63521", "passed": true }, { "field": "maturity_date", "actual": "2027-01-15", "expected": "2027-01-15", "passed": true }, { "field": "original_principal_usd", "actual": 8640000, "expected": 8640000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Casey Elm1", "expected": "Casey Elm1", "passed": true }, { "field": "contact_email", "actual": "finance1@refinance-leads-elm.example", "expected": "finance1@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968", "S-R4437" ] }, { "record_id": "R7631", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REF-53828", "expected": "ELM-REF-53828", "passed": true }, { "field": "owner_entity_id", "actual": "O-ELM-REF-53828", "expected": "O-ELM-REF-53828", "passed": true }, { "field": "maturity_date", "actual": "2027-03-15", "expected": "2027-03-15", "passed": true }, { "field": "original_principal_usd", "actual": 9936000, "expected": 9936000, "passed": true }, { "field": "outstanding_balance_usd", "actual": null, "expected": null, "passed": true }, { "field": "contact_name", "actual": "Avery Elm3", "expected": "Avery Elm3", "passed": true }, { "field": "contact_email", "actual": "finance3@refinance-leads-elm.example", "expected": "finance3@refinance-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7631" ], "reported_sources": [ "S-R7631" ] } ], "decision_checks": [ { "record_id": "R7798", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1548", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R5893", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6909", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2801", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R7631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4392", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4755", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R3427", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4641", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R3478", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9095", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R9201", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2376", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R1927", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R9233", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8058", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R2630", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4437", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-fir", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:41:13.418Z", "observed_completed_at": "2026-09-07T02:44:55.707Z", "packet_sha256": "4325bb9d69c6735a0780891406199f9072d97f316ff6d477e61f895b81216d17", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/refinance_leads_fir_dallas_industrial.csv", "sha256": "d1908b6b2cd20e0c9497d5c075316a3a1e35d9703a5dcde5346ab176b920c703", "bytes": 136 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.5908495, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 128.89, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "9041a7fc-abf0-4b2e-9494-38b30ff21d6c", "session_id": "98ee80db-026e-4997-b810-c7cc07845f32", "started_at": "2026-09-06T22:42:55.441000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.43179125, 0.15905825 ], "total_cost_usd": 0.5908495, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 128.89, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/refinance-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-fir", "task": "refinance_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R2813", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2663", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7919", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4743", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R4927", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R5151", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1776", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7702", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R5451", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7436", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8900", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market", "Maturity outside window" ] }, { "record_id": "R5181", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3402", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7201", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9169", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R7833", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1118", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1203", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R9426", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5442", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/refinance_leads_fir_dallas_industrial.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "d1908b6b2cd20e0c9497d5c075316a3a1e35d9703a5dcde5346ab176b920c703" } }, { "case_id": "refinance-leads-fir", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:44:01.714217+00:00", "completed_at": "2026-09-07T02:46:52.344608+00:00", "elapsed_seconds": 170.63, "cost_usd": 0.18099425, "known_partial_cost_usd": 0.18099425, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8238, "completion_tokens": 4513, "total_tokens": 12751, "cost": 0.0554275, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4416, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0554275, "gateway_cost": 0.0554275 }, "finish_reason": "tool_calls", "elapsed_seconds": 48.381, "completed_at": "2026-09-07T02:44:50.108204+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8317, "completion_tokens": 5682, "total_tokens": 13999, "cost": 0.05800025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8192, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5248, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05800025, "gateway_cost": 0.05800025 }, "finish_reason": "tool_calls", "elapsed_seconds": 58.507, "completed_at": "2026-09-07T02:45:48.628796+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8738, "completion_tokens": 6586, "total_tokens": 15324, "cost": 0.0675665, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8192, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5504, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0675665, "gateway_cost": 0.0675665 }, "finish_reason": "tool_calls", "elapsed_seconds": 63.084, "completed_at": "2026-09-07T02:46:52.342147+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "d1908b6b2cd20e0c9497d5c075316a3a1e35d9703a5dcde5346ab176b920c703", "bytes": 136 }, { "path": "artifacts/underwriting-v1.xlsx", "sha256": "6488b816464685b19cb8a7cbaad4c40e16ae817106706cc9b598617222fc86af", "bytes": 4540 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/refinance-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-fir", "task": "refinance_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R2813", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2663", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7919", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4743", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R4927", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R5151", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1776", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7702", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R5451", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7436", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8900", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market", "Maturity outside window" ] }, { "record_id": "R5181", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3402", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7201", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9169", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R7833", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1118", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1203", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R9426", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5442", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "d1908b6b2cd20e0c9497d5c075316a3a1e35d9703a5dcde5346ab176b920c703" } }, { "case_id": "refinance-leads-fir", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:44:56.105872+00:00", "completed_at": "2026-09-07T02:45:50.957196+00:00", "elapsed_seconds": 54.851, "cost_usd": 0.25638, "known_partial_cost_usd": 0.25638, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12824, "completion_tokens": 1797, "total_tokens": 14621, "cost": 0.109045, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1251, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.109045, "gateway_cost": 0.109045 }, "finish_reason": "tool_calls", "elapsed_seconds": 21.456, "completed_at": "2026-09-07T02:45:17.583481+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13432, "completion_tokens": 3207, "total_tokens": 16639, "cost": 0.147335, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 704, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.147335, "gateway_cost": 0.147335 }, "finish_reason": "stop", "elapsed_seconds": 33.369, "completed_at": "2026-09-07T02:45:50.955726+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "42b81c273d32f4a185e19f0d340adefa9da09475b97a9302dcc6093868b43a38", "bytes": 710 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/refinance-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-fir", "task": "refinance_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R2813", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2663", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7919", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4743", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R4927", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R5151", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1776", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7702", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R5451", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7436", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8900", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market", "Maturity outside window" ] }, { "record_id": "R5181", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3402", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7201", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9169", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R7833", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1118", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1203", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R9426", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5442", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "42b81c273d32f4a185e19f0d340adefa9da09475b97a9302dcc6093868b43a38" } }, { "case_id": "refinance-leads-fir", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:45:50.958101+00:00", "completed_at": "2026-09-07T02:47:15.693673+00:00", "elapsed_seconds": 84.735, "cost_usd": 0.069485, "known_partial_cost_usd": 0.069485, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7492, "completion_tokens": 6012, "total_tokens": 13504, "cost": 0.069485, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4480, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.069485, "gateway_cost": 0.069485 }, "finish_reason": "stop", "elapsed_seconds": 84.716, "completed_at": "2026-09-07T02:47:15.692263+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/refinance-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-fir", "task": "refinance_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R2813", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2663", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7919", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4743", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R4927", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R5151", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1776", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7702", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R5451", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7436", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8900", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market", "Maturity outside window" ] }, { "record_id": "R5181", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3402", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7201", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9169", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R7833", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1118", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1203", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R9426", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5442", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "refinance-leads-fir", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:46:52.345626+00:00", "completed_at": "2026-09-07T02:47:32.572238+00:00", "elapsed_seconds": 40.226, "cost_usd": 0.15002, "known_partial_cost_usd": 0.15002, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10539, "completion_tokens": 3893, "total_tokens": 14432, "cost": 0.15002, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 732, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.15002, "gateway_cost": 0.15002 }, "finish_reason": "stop", "elapsed_seconds": 40.21, "completed_at": "2026-09-07T02:47:32.570580+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "refinance_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/refinance-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "refinance-leads-fir", "task": "refinance_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R2813", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence" ] }, { "record_id": "R2663", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7919", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4743", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity not confirmed by executed evidence", "Unresolved extension" ] }, { "record_id": "R4927", "actual": false, "expected": false, "passed": true, "reasons": [ "Not the record owner" ] }, { "record_id": "R5151", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1776", "actual": false, "expected": false, "passed": true, "reasons": [ "Maturity outside window" ] }, { "record_id": "R7702", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R5451", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7436", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R8900", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market", "Maturity outside window" ] }, { "record_id": "R5181", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3402", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7201", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9169", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R7833", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1118", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1203", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale ownership verification" ] }, { "record_id": "R9426", "actual": false, "expected": false, "passed": true, "reasons": [ "Loan status not confirmed active" ] }, { "record_id": "R5442", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-alder", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:26:49.247Z", "observed_completed_at": "2026-09-07T02:29:45.662Z", "packet_sha256": "09f378dcadb00b2b50473ce03017cbecb65853a1428debf7fb327198e3441fa4", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/rent_comps_charlotte_multifamily.xlsx", "sha256": "acb8801df50d6944a9fb300cb74757900559ab3ad72e98932818cdfdd1fa08d1", "bytes": 8999 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.56936725, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 126.66, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "07bc8343-2410-4cee-a40c-e0bc85ed3e6b", "session_id": "ddbf607c-e06e-401b-b4e5-ca01cd93ae2d", "started_at": "2026-09-06T22:28:29.160000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.401664, 0.16770325 ], "total_cost_usd": 0.56936725, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 126.66, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/rent-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-alder", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1927", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1927" ], "reported_sources": [ "S-R1927" ] }, { "record_id": "R9957", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9957", "U-R9957-1" ], "reported_sources": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2302" ], "reported_sources": [ "S-R2302" ] }, { "record_id": "R6740", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6740" ], "reported_sources": [ "S-R6740" ] }, { "record_id": "R6049", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6049" ], "reported_sources": [ "S-R6049" ] } ], "decision_checks": [ { "record_id": "R8015", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9957", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6049", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2302", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6740", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4257", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4144", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4517", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R7480", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R6844", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5477", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R8781", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1298", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7775", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3818", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8821", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2138", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2543", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2337", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/rent_comps_charlotte_multifamily.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid Shortlist", "Candidate Decisions", "Notes" ], "table_sheet": "Comp Grid Shortlist", "original_record_order": [ "R1927", "R9957", "R2302", "R6740", "R6049" ], "answer_record_order": [ "R1927", "R9957", "R2302", "R6740", "R6049" ], "field_checks": [ { "record_id": "R1927", "field": "entity_id", "actual": "ALDER-REN-19075", "original": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "record_id": "R1927", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1927", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R1927", "field": "effective_net_rent_psf_year", "actual": 11.2794, "original": 11.2794, "expected": 11.28, "passed": true }, { "record_id": "R1927", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R9957", "field": "entity_id", "actual": "ALDER-REN-19383", "original": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "record_id": "R9957", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9957", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R9957", "field": "effective_net_rent_psf_year", "actual": 16.9256, "original": 16.9256, "expected": 16.93, "passed": true }, { "record_id": "R9957", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R2302", "field": "entity_id", "actual": "ALDER-REN-80164", "original": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "record_id": "R2302", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2302", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R2302", "field": "effective_net_rent_psf_year", "actual": 19.4053, "original": 19.4053, "expected": 19.41, "passed": true }, { "record_id": "R2302", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R6740", "field": "entity_id", "actual": "ALDER-REN-24854", "original": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "record_id": "R6740", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R6740", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R6740", "field": "effective_net_rent_psf_year", "actual": 9.468, "original": 9.468, "expected": 9.47, "passed": true }, { "record_id": "R6740", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R6049", "field": "entity_id", "actual": "ALDER-REN-90122", "original": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "record_id": "R6049", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6049", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R6049", "field": "effective_net_rent_psf_year", "actual": 22.1227, "original": 22.1227, "expected": 22.12, "passed": true }, { "record_id": "R6049", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1927", "passed": true, "expected": [ "S-R1927" ], "actual": [ "S-R1927" ] }, { "record_id": "R9957", "passed": true, "expected": [ "S-R9957", "U-R9957-1" ], "actual": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "passed": true, "expected": [ "S-R2302" ], "actual": [ "S-R2302", "U-R2302-2" ] }, { "record_id": "R6740", "passed": true, "expected": [ "S-R6740" ], "actual": [ "S-R6740", "U-R6740-3" ] }, { "record_id": "R6049", "passed": true, "expected": [ "S-R6049" ], "actual": [ "S-R6049" ] } ], "source_sha256": "acb8801df50d6944a9fb300cb74757900559ab3ad72e98932818cdfdd1fa08d1" } }, { "case_id": "rent-comps-alder", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:47:15.694559+00:00", "completed_at": "2026-09-07T02:56:09.775527+00:00", "elapsed_seconds": 534.079, "cost_usd": 0.4952655, "known_partial_cost_usd": 0.4952655, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 8, "tool_calls": 8, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5626, "completion_tokens": 7139, "total_tokens": 12765, "cost": 0.0784225, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7104, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0784225, "gateway_cost": 0.0784225 }, "finish_reason": "tool_calls", "elapsed_seconds": 81.668, "completed_at": "2026-09-07T02:48:37.382882+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5669, "completion_tokens": 4522, "total_tokens": 10191, "cost": 0.04611425, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5504, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4352, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.04611425, "gateway_cost": 0.04611425 }, "finish_reason": "tool_calls", "elapsed_seconds": 50.867, "completed_at": "2026-09-07T02:49:28.254328+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5905, "completion_tokens": 3864, "total_tokens": 9769, "cost": 0.03968525, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5632, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3648, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.03968525, "gateway_cost": 0.03968525 }, "finish_reason": "tool_calls", "elapsed_seconds": 49.072, "completed_at": "2026-09-07T02:50:17.329361+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6149, "completion_tokens": 4537, "total_tokens": 10686, "cost": 0.04643225, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5888, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4096, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.04643225, "gateway_cost": 0.04643225 }, "finish_reason": "tool_calls", "elapsed_seconds": 56.48, "completed_at": "2026-09-07T02:51:13.812140+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6798, "completion_tokens": 6964, "total_tokens": 13762, "cost": 0.0713695, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6016, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6272, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0713695, "gateway_cost": 0.0713695 }, "finish_reason": "tool_calls", "elapsed_seconds": 75.735, "completed_at": "2026-09-07T02:52:29.550750+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7869, "completion_tokens": 5453, "total_tokens": 13322, "cost": 0.05687825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6656, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5056, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05687825, "gateway_cost": 0.05687825 }, "finish_reason": "tool_calls", "elapsed_seconds": 58.027, "completed_at": "2026-09-07T02:53:27.593484+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8415, "completion_tokens": 6910, "total_tokens": 15325, "cost": 0.07083475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7808, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5696, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07083475, "gateway_cost": 0.07083475 }, "finish_reason": "tool_calls", "elapsed_seconds": 67.522, "completed_at": "2026-09-07T02:54:35.118538+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9639, "completion_tokens": 8284, "total_tokens": 17923, "cost": 0.08552875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8320, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6464, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08552875, "gateway_cost": 0.08552875 }, "finish_reason": "tool_calls", "elapsed_seconds": 94.137, "completed_at": "2026-09-07T02:56:09.773109+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "cafc2706f68d42ca21238fa7f442208c0a0ffffde2d096267eb699be4ff27780", "bytes": 4461 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/rent-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-alder", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1298", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1298" ], "reported_sources": [ "S-R1298" ] }, { "record_id": "R9957", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.9256126563, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9957", "U-R9957-1" ], "reported_sources": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.4052715573, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2302" ], "reported_sources": [ "S-R2302" ] }, { "record_id": "R6740", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468025, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6740" ], "reported_sources": [ "S-R6740" ] }, { "record_id": "R6049", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.1227195313, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6049" ], "reported_sources": [ "S-R6049" ] } ], "decision_checks": [ { "record_id": "R8015", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9957", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6049", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2302", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6740", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4257", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4144", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4517", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R7480", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R6844", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5477", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R8781", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1298", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7775", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3818", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8821", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2138", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2543", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2337", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R1298", "R9957", "R2302", "R6740", "R6049" ], "answer_record_order": [ "R1298", "R9957", "R2302", "R6740", "R6049" ], "field_checks": [ { "record_id": "R1298", "field": "entity_id", "actual": "ALDER-REN-19075", "original": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "record_id": "R1298", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1298", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21, "passed": true }, { "record_id": "R1298", "field": "effective_net_rent_psf_year", "actual": 11.279374999999998, "original": 11.279374999999998, "expected": 11.279375, "passed": true }, { "record_id": "R1298", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R9957", "field": "entity_id", "actual": "ALDER-REN-19383", "original": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "record_id": "R9957", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9957", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R9957", "field": "effective_net_rent_psf_year", "actual": 16.925612656249996, "original": 16.925612656249996, "expected": 16.9256126563, "passed": true }, { "record_id": "R9957", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R2302", "field": "entity_id", "actual": "ALDER-REN-80164", "original": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "record_id": "R2302", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2302", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R2302", "field": "effective_net_rent_psf_year", "actual": 19.40527155729166, "original": 19.40527155729166, "expected": 19.4052715573, "passed": true }, { "record_id": "R2302", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R6740", "field": "entity_id", "actual": "ALDER-REN-24854", "original": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "record_id": "R6740", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R6740", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R6740", "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "original": 9.468024999999992, "expected": 9.468025, "passed": true }, { "record_id": "R6740", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R6049", "field": "entity_id", "actual": "ALDER-REN-90122", "original": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "record_id": "R6049", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6049", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R6049", "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "original": 22.122719531249995, "expected": 22.1227195313, "passed": true }, { "record_id": "R6049", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1298", "passed": true, "expected": [ "S-R1298" ], "actual": [ "S-R1298" ] }, { "record_id": "R9957", "passed": true, "expected": [ "S-R9957", "U-R9957-1" ], "actual": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "passed": true, "expected": [ "S-R2302" ], "actual": [ "S-R2302" ] }, { "record_id": "R6740", "passed": true, "expected": [ "S-R6740" ], "actual": [ "S-R6740" ] }, { "record_id": "R6049", "passed": true, "expected": [ "S-R6049" ], "actual": [ "S-R6049" ] } ], "source_sha256": "cafc2706f68d42ca21238fa7f442208c0a0ffffde2d096267eb699be4ff27780" } }, { "case_id": "rent-comps-alder", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:47:27.730419+00:00", "completed_at": "2026-09-07T02:49:46.786908+00:00", "elapsed_seconds": 139.056, "cost_usd": 0.50453, "known_partial_cost_usd": 0.50453, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8939, "completion_tokens": 2447, "total_tokens": 11386, "cost": 0.10587, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2023, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10587, "gateway_cost": 0.10587 }, "finish_reason": "tool_calls", "elapsed_seconds": 29.757, "completed_at": "2026-09-07T02:47:57.502391+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9477, "completion_tokens": 6738, "total_tokens": 16215, "cost": 0.215835, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2150, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.215835, "gateway_cost": 0.215835 }, "finish_reason": "tool_calls", "elapsed_seconds": 62.709, "completed_at": "2026-09-07T02:49:00.214725+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 14295, "completion_tokens": 4454, "total_tokens": 18749, "cost": 0.182825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 591, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.182825, "gateway_cost": 0.182825 }, "finish_reason": "tool_calls", "elapsed_seconds": 46.035, "completed_at": "2026-09-07T02:49:46.784626+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "a7c0f2c89ff6d86bcc582ae6efc9414bb85f5247c27c764c98fbb36ddd37db02", "bytes": 5600 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/rent-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-alder", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1298", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1298" ], "reported_sources": [ "S-R1298" ] }, { "record_id": "R9957", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9957", "U-R9957-1" ], "reported_sources": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2302" ], "reported_sources": [ "S-R2302" ] }, { "record_id": "R6740", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6740" ], "reported_sources": [ "S-R6740" ] }, { "record_id": "R6049", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6049" ], "reported_sources": [ "S-R6049" ] } ], "decision_checks": [ { "record_id": "R8015", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9957", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6049", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2302", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6740", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4257", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4144", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4517", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R7480", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R6844", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5477", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R8781", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1298", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7775", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3818", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8821", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2138", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2543", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2337", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R1298", "R9957", "R2302", "R6740", "R6049" ], "answer_record_order": [ "R1298", "R9957", "R2302", "R6740", "R6049" ], "field_checks": [ { "record_id": "R1298", "field": "entity_id", "actual": "ALDER-REN-19075", "original": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "record_id": "R1298", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1298", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21, "passed": true }, { "record_id": "R1298", "field": "effective_net_rent_psf_year", "actual": 11.279374999999998, "original": 11.279374999999998, "expected": 11.28, "passed": true }, { "record_id": "R1298", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R9957", "field": "entity_id", "actual": "ALDER-REN-19383", "original": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "record_id": "R9957", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9957", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R9957", "field": "effective_net_rent_psf_year", "actual": 16.925612656249996, "original": 16.925612656249996, "expected": 16.93, "passed": true }, { "record_id": "R9957", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R2302", "field": "entity_id", "actual": "ALDER-REN-80164", "original": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "record_id": "R2302", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2302", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R2302", "field": "effective_net_rent_psf_year", "actual": 19.40527155729166, "original": 19.40527155729166, "expected": 19.41, "passed": true }, { "record_id": "R2302", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R6740", "field": "entity_id", "actual": "ALDER-REN-24854", "original": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "record_id": "R6740", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R6740", "field": "annual_face_rent_psf", "actual": 24.119999999999997, "original": 24.119999999999997, "expected": 24.12, "passed": true }, { "record_id": "R6740", "field": "effective_net_rent_psf_year", "actual": 9.468024999999997, "original": 9.468024999999997, "expected": 9.47, "passed": true }, { "record_id": "R6740", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R6049", "field": "entity_id", "actual": "ALDER-REN-90122", "original": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "record_id": "R6049", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6049", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R6049", "field": "effective_net_rent_psf_year", "actual": 22.12271953125, "original": 22.12271953125, "expected": 22.12, "passed": true }, { "record_id": "R6049", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1298", "passed": true, "expected": [ "S-R1298" ], "actual": [ "S-R1298" ] }, { "record_id": "R9957", "passed": true, "expected": [ "S-R9957", "U-R9957-1" ], "actual": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "passed": true, "expected": [ "S-R2302" ], "actual": [ "S-R2302", "U-R2302-2" ] }, { "record_id": "R6740", "passed": true, "expected": [ "S-R6740" ], "actual": [ "S-R6740", "U-R6740-3" ] }, { "record_id": "R6049", "passed": true, "expected": [ "S-R6049" ], "actual": [ "S-R6049" ] } ], "source_sha256": "a7c0f2c89ff6d86bcc582ae6efc9414bb85f5247c27c764c98fbb36ddd37db02" } }, { "case_id": "rent-comps-alder", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:47:32.573086+00:00", "completed_at": "2026-09-07T02:50:47.775936+00:00", "elapsed_seconds": 195.202, "cost_usd": 0.14403, "known_partial_cost_usd": 0.14403, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4880, "completion_tokens": 13793, "total_tokens": 18673, "cost": 0.14403, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 11200, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.14403, "gateway_cost": 0.14403 }, "finish_reason": "stop", "elapsed_seconds": 195.184, "completed_at": "2026-09-07T02:50:47.774373+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/rent-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-alder", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1298", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1298" ], "reported_sources": [ "S-R1298" ] }, { "record_id": "R9957", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9957", "U-R9957-1" ], "reported_sources": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.4052715572, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2302" ], "reported_sources": [ "S-R2302" ] }, { "record_id": "R6740", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.467858333333, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6740" ], "reported_sources": [ "S-R6740" ] }, { "record_id": "R6049", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12271953125, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6049" ], "reported_sources": [ "S-R6049" ] } ], "decision_checks": [ { "record_id": "R8015", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9957", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6049", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2302", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6740", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4257", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4144", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4517", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R7480", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R6844", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5477", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R8781", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1298", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7775", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3818", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8821", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2138", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2543", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2337", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-alder", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:49:46.787854+00:00", "completed_at": "2026-09-07T02:51:02.096060+00:00", "elapsed_seconds": 75.308, "cost_usd": 0.215745, "known_partial_cost_usd": 0.215745, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 6654, "completion_tokens": 7299, "total_tokens": 13953, "cost": 0.215745, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2650, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.215745, "gateway_cost": 0.215745 }, "finish_reason": "stop", "elapsed_seconds": 75.286, "completed_at": "2026-09-07T02:51:02.094540+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/rent-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-alder", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1298", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19075", "expected": "ALDER-REN-19075", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1298" ], "reported_sources": [ "S-R1298" ] }, { "record_id": "R9957", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-19383", "expected": "ALDER-REN-19383", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9957", "U-R9957-1" ], "reported_sources": [ "S-R9957", "U-R9957-1" ] }, { "record_id": "R2302", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-80164", "expected": "ALDER-REN-80164", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2302" ], "reported_sources": [ "S-R2302" ] }, { "record_id": "R6740", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-24854", "expected": "ALDER-REN-24854", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6740" ], "reported_sources": [ "S-R6740" ] }, { "record_id": "R6049", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-REN-90122", "expected": "ALDER-REN-90122", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6049" ], "reported_sources": [ "S-R6049" ] } ], "decision_checks": [ { "record_id": "R8015", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1927", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9957", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6049", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2302", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6740", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4257", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4144", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4517", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R7480", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R6844", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5477", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R8781", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1298", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7775", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3818", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8821", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2138", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2543", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2337", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-birch", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:30:42.927Z", "observed_completed_at": "2026-09-07T02:33:15.080Z", "packet_sha256": "26b6483dae943726636203505622b7928fbfc897724db3d82daa1c98a40193c7", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/birch_comp_grid.xlsx", "sha256": "390d695d49961a2f54d35c1873d6b258eaf3a6722a6a32e110834b7b9dbb0043", "bytes": 7768 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.55367875, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 116.42, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "ed1c3aad-5fad-412e-835a-bf9ca097fa2b", "session_id": "9122db29-8b6f-48f5-89ab-fc3863cb6363", "started_at": "2026-09-06T22:32:08.550000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.3779745, 0.17570425 ], "total_cost_usd": 0.55367875, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 116.42, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/rent-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-birch", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R2614", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2614" ], "reported_sources": [ "S-R2614" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.926, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353", "U-R3353-1" ], "reported_sources": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.405, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2434" ], "reported_sources": [ "S-R2434" ] }, { "record_id": "R1930", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1930" ], "reported_sources": [ "S-R1930" ] }, { "record_id": "R6289", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.123, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6289" ], "reported_sources": [ "S-R6289" ] } ], "decision_checks": [ { "record_id": "R7254", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8772", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2614", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8177", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6289", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3442", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R5629", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1930", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1009", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2434", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9966", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7377", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R1695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9256", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5471", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7345", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R3964", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5747", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7562", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/birch_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions" ], "table_sheet": "Comp Grid", "original_record_order": [ "R2614", "R3353", "R2434", "R1930", "R6289" ], "answer_record_order": [ "R2614", "R3353", "R2434", "R1930", "R6289" ], "field_checks": [ { "record_id": "R2614", "field": "entity_id", "actual": "BIRCH-REN-46690", "original": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "record_id": "R2614", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R2614", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R2614", "field": "effective_net_rent_psf_year", "actual": 11.279, "original": 11.279, "expected": 11.279, "passed": true }, { "record_id": "R2614", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R3353", "field": "entity_id", "actual": "BIRCH-REN-48467", "original": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "record_id": "R3353", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R3353", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R3353", "field": "effective_net_rent_psf_year", "actual": 16.926, "original": 16.926, "expected": 16.926, "passed": true }, { "record_id": "R3353", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R2434", "field": "entity_id", "actual": "BIRCH-REN-81113", "original": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "record_id": "R2434", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2434", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R2434", "field": "effective_net_rent_psf_year", "actual": 19.405, "original": 19.405, "expected": 19.405, "passed": true }, { "record_id": "R2434", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R1930", "field": "entity_id", "actual": "BIRCH-REN-15479", "original": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "record_id": "R1930", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R1930", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R1930", "field": "effective_net_rent_psf_year", "actual": 9.468, "original": 9.468, "expected": 9.468, "passed": true }, { "record_id": "R1930", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R6289", "field": "entity_id", "actual": "BIRCH-REN-11670", "original": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "record_id": "R6289", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6289", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R6289", "field": "effective_net_rent_psf_year", "actual": 22.123, "original": 22.123, "expected": 22.123, "passed": true }, { "record_id": "R6289", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2614", "passed": true, "expected": [ "S-R2614" ], "actual": [ "S-R2614" ] }, { "record_id": "R3353", "passed": true, "expected": [ "S-R3353", "U-R3353-1" ], "actual": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "passed": true, "expected": [ "S-R2434" ], "actual": [ "S-R2434" ] }, { "record_id": "R1930", "passed": true, "expected": [ "S-R1930" ], "actual": [ "S-R1930" ] }, { "record_id": "R6289", "passed": true, "expected": [ "S-R6289" ], "actual": [ "S-R6289" ] } ], "source_sha256": "390d695d49961a2f54d35c1873d6b258eaf3a6722a6a32e110834b7b9dbb0043" } }, { "case_id": "rent-comps-birch", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:50:47.776883+00:00", "completed_at": "2026-09-07T02:58:36.757423+00:00", "elapsed_seconds": 468.979, "cost_usd": 0.45746025, "known_partial_cost_usd": 0.45746025, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 11, "tool_calls": 11, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5624, "completion_tokens": 3112, "total_tokens": 8736, "cost": 0.03815, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2944, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.03815, "gateway_cost": 0.03815 }, "finish_reason": "tool_calls", "elapsed_seconds": 45.94, "completed_at": "2026-09-07T02:51:33.736072+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5795, "completion_tokens": 3099, "total_tokens": 8894, "cost": 0.03204175, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5504, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2880, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.03204175, "gateway_cost": 0.03204175 }, "finish_reason": "tool_calls", "elapsed_seconds": 34.491, "completed_at": "2026-09-07T02:52:08.229932+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6121, "completion_tokens": 5009, "total_tokens": 11130, "cost": 0.05126125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5760, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3840, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05126125, "gateway_cost": 0.05126125 }, "finish_reason": "tool_calls", "elapsed_seconds": 51.91, "completed_at": "2026-09-07T02:53:00.143963+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7281, "completion_tokens": 5355, "total_tokens": 12636, "cost": 0.05588325, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6016, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4032, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05588325, "gateway_cost": 0.05588325 }, "finish_reason": "tool_calls", "elapsed_seconds": 49.145, "completed_at": "2026-09-07T02:53:49.292225+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8563, "completion_tokens": 3332, "total_tokens": 11895, "cost": 0.03595975, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7168, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3136, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.03595975, "gateway_cost": 0.03595975 }, "finish_reason": "tool_calls", "elapsed_seconds": 35.505, "completed_at": "2026-09-07T02:54:24.800040+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8864, "completion_tokens": 3600, "total_tokens": 12464, "cost": 0.037576, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8448, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3392, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.037576, "gateway_cost": 0.037576 }, "finish_reason": "tool_calls", "elapsed_seconds": 34.152, "completed_at": "2026-09-07T02:54:58.955899+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9157, "completion_tokens": 255, "total_tokens": 9412, "cost": 0.00406025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 8832, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 0, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.00406025, "gateway_cost": 0.00406025 }, "finish_reason": "tool_calls", "elapsed_seconds": 4.501, "completed_at": "2026-09-07T02:55:03.460744+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9567, "completion_tokens": 229, "total_tokens": 9796, "cost": 0.00402475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9088, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 0, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.00402475, "gateway_cost": 0.00402475 }, "finish_reason": "tool_calls", "elapsed_seconds": 3.382, "completed_at": "2026-09-07T02:55:06.846105+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9941, "completion_tokens": 8750, "total_tokens": 18691, "cost": 0.08927025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9472, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6976, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08927025, "gateway_cost": 0.08927025 }, "finish_reason": "tool_calls", "elapsed_seconds": 98.491, "completed_at": "2026-09-07T02:56:45.342932+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 11720, "completion_tokens": 284, "total_tokens": 12004, "cost": 0.006402, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9856, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 0, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.006402, "gateway_cost": 0.006402 }, "finish_reason": "tool_calls", "elapsed_seconds": 3.62, "completed_at": "2026-09-07T02:56:49.483269+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 12028, "completion_tokens": 10090, "total_tokens": 22118, "cost": 0.102831, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 11648, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 8128, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.102831, "gateway_cost": 0.102831 }, "finish_reason": "tool_calls", "elapsed_seconds": 107.267, "completed_at": "2026-09-07T02:58:36.754894+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "659339be3aae8a4ff51c8729bc62c70a2b4a313dac98a0b653fc31e822bdafc2", "bytes": 503 }, { "path": "artifacts/underwriting-v1.xlsx", "sha256": "891c3b3df072685a8bd23d57a41af20ad396d152b57d59decad60ded1b0fbdd5", "bytes": 4628 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/rent-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-birch", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1695", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1695" ], "reported_sources": [ "S-R1695", "S-R2614" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353", "U-R3353-1" ], "reported_sources": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2434" ], "reported_sources": [ "S-R2434" ] }, { "record_id": "R1930", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1930" ], "reported_sources": [ "S-R1930" ] }, { "record_id": "R6289", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6289" ], "reported_sources": [ "S-R6289" ] } ], "decision_checks": [ { "record_id": "R7254", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8772", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2614", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8177", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6289", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3442", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R5629", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1930", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1009", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2434", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9966", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7377", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R1695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9256", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5471", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7345", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R3964", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5747", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7562", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comps" ], "table_sheet": "Comps", "original_record_order": [ "R1695", "R3353", "R2434", "R1930", "R6289" ], "answer_record_order": [ "R1695", "R3353", "R2434", "R1930", "R6289" ], "field_checks": [ { "record_id": "R1695", "field": "entity_id", "actual": "BIRCH-REN-46690", "original": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "record_id": "R1695", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1695", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21, "passed": true }, { "record_id": "R1695", "field": "effective_net_rent_psf_year", "actual": 11.279374999999998, "original": 11.279374999999998, "expected": 11.279375, "passed": true }, { "record_id": "R1695", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R3353", "field": "entity_id", "actual": "BIRCH-REN-48467", "original": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "record_id": "R3353", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R3353", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R3353", "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "original": 16.92561265625, "expected": 16.92561265625, "passed": true }, { "record_id": "R3353", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R2434", "field": "entity_id", "actual": "BIRCH-REN-81113", "original": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "record_id": "R2434", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2434", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R2434", "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "original": 19.405271557291666, "expected": 19.405271557291666, "passed": true }, { "record_id": "R2434", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R1930", "field": "entity_id", "actual": "BIRCH-REN-15479", "original": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "record_id": "R1930", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R1930", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R1930", "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "original": 9.468024999999992, "expected": 9.468024999999992, "passed": true }, { "record_id": "R1930", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R6289", "field": "entity_id", "actual": "BIRCH-REN-11670", "original": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "record_id": "R6289", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6289", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R6289", "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "original": 22.122719531249995, "expected": 22.122719531249995, "passed": true }, { "record_id": "R6289", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1695", "passed": true, "expected": [ "S-R1695", "S-R2614" ], "actual": [ "S-R1695", "S-R2614" ] }, { "record_id": "R3353", "passed": true, "expected": [ "S-R3353", "U-R3353-1" ], "actual": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "passed": true, "expected": [ "S-R2434" ], "actual": [ "S-R2434" ] }, { "record_id": "R1930", "passed": true, "expected": [ "S-R1930" ], "actual": [ "S-R1930" ] }, { "record_id": "R6289", "passed": true, "expected": [ "S-R6289" ], "actual": [ "S-R6289" ] } ], "source_sha256": "891c3b3df072685a8bd23d57a41af20ad396d152b57d59decad60ded1b0fbdd5" } }, { "case_id": "rent-comps-birch", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:51:02.096918+00:00", "completed_at": "2026-09-07T02:53:58.631496+00:00", "elapsed_seconds": 176.534, "cost_usd": 0.6280600000000001, "known_partial_cost_usd": 0.6280600000000001, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8941, "completion_tokens": 2553, "total_tokens": 11494, "cost": 0.10853, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2075, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10853, "gateway_cost": 0.10853 }, "finish_reason": "tool_calls", "elapsed_seconds": 30.198, "completed_at": "2026-09-07T02:51:32.313680+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9532, "completion_tokens": 11260, "total_tokens": 20792, "cost": 0.32916, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3691, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.32916, "gateway_cost": 0.32916 }, "finish_reason": "tool_calls", "elapsed_seconds": 104.451, "completed_at": "2026-09-07T02:53:16.769165+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 17309, "completion_tokens": 4153, "total_tokens": 21462, "cost": 0.19037, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 160, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.19037, "gateway_cost": 0.19037 }, "finish_reason": "stop", "elapsed_seconds": 41.312, "completed_at": "2026-09-07T02:53:58.629848+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "273569cf5b740c842cff58d2df783d358e38323931130d6b229d56f2f6459768", "bytes": 6437 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/rent-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-birch", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 7, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1695", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1695" ], "reported_sources": [ "S-R1695" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353", "U-R3353-1" ], "reported_sources": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2434" ], "reported_sources": [ "S-R2434" ] }, { "record_id": "R1930", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1930" ], "reported_sources": [ "S-R1930" ] }, { "record_id": "R6289", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6289" ], "reported_sources": [ "S-R6289" ] } ], "decision_checks": [ { "record_id": "R7254", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8772", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2614", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8177", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6289", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3442", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R5629", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1930", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1009", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2434", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9966", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7377", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R1695", "actual": null, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R9256", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5471", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7345", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R3964", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5747", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7562", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R1695", "R3353", "R2434", "R1930", "R6289" ], "answer_record_order": [ "R1695", "R3353", "R2434", "R1930", "R6289" ], "field_checks": [ { "record_id": "R1695", "field": "entity_id", "actual": "BIRCH-REN-46690", "original": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "record_id": "R1695", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1695", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R1695", "field": "effective_net_rent_psf_year", "actual": 11.279374999999998, "original": 11.279374999999998, "expected": 11.28, "passed": true }, { "record_id": "R1695", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R3353", "field": "entity_id", "actual": "BIRCH-REN-48467", "original": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "record_id": "R3353", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R3353", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R3353", "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "original": 16.92561265625, "expected": 16.93, "passed": true }, { "record_id": "R3353", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R2434", "field": "entity_id", "actual": "BIRCH-REN-81113", "original": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "record_id": "R2434", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2434", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R2434", "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "original": 19.405271557291666, "expected": 19.41, "passed": true }, { "record_id": "R2434", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R1930", "field": "entity_id", "actual": "BIRCH-REN-15479", "original": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "record_id": "R1930", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R1930", "field": "annual_face_rent_psf", "actual": 24.119999999999997, "original": 24.119999999999997, "expected": 24.12, "passed": true }, { "record_id": "R1930", "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "original": 9.468024999999992, "expected": 9.47, "passed": true }, { "record_id": "R1930", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R6289", "field": "entity_id", "actual": "BIRCH-REN-11670", "original": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "record_id": "R6289", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6289", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R6289", "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "original": 22.122719531249995, "expected": 22.12, "passed": true }, { "record_id": "R6289", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1695", "passed": true, "expected": [ "S-R1695" ], "actual": [ "S-R1695", "S-R2614" ] }, { "record_id": "R3353", "passed": true, "expected": [ "S-R3353", "U-R3353-1" ], "actual": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "passed": true, "expected": [ "S-R2434" ], "actual": [ "S-R2434", "U-R2434-2" ] }, { "record_id": "R1930", "passed": true, "expected": [ "S-R1930" ], "actual": [ "S-R1930", "U-R1930-3" ] }, { "record_id": "R6289", "passed": true, "expected": [ "S-R6289" ], "actual": [ "S-R6289" ] } ], "source_sha256": "273569cf5b740c842cff58d2df783d358e38323931130d6b229d56f2f6459768" } }, { "case_id": "rent-comps-birch", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:53:58.633012+00:00", "completed_at": "2026-09-07T02:56:02.680234+00:00", "elapsed_seconds": 124.047, "cost_usd": 0.1338575, "known_partial_cost_usd": 0.1338575, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4878, "completion_tokens": 12776, "total_tokens": 17654, "cost": 0.1338575, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 10624, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.1338575, "gateway_cost": 0.1338575 }, "finish_reason": "stop", "elapsed_seconds": 124.027, "completed_at": "2026-09-07T02:56:02.678981+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/rent-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-birch", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1695", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.2794, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1695" ], "reported_sources": [ "S-R1695", "S-R2614" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.9226, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353", "U-R3353-1" ], "reported_sources": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.4053, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2434" ], "reported_sources": [ "S-R2434" ] }, { "record_id": "R1930", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1930" ], "reported_sources": [ "S-R1930" ] }, { "record_id": "R6289", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.1227, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6289" ], "reported_sources": [ "S-R6289" ] } ], "decision_checks": [ { "record_id": "R7254", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8772", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2614", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8177", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6289", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3442", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R5629", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1930", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1009", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2434", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9966", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7377", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R1695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9256", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5471", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7345", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R3964", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5747", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7562", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-birch", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:56:02.681160+00:00", "completed_at": "2026-09-07T02:57:19.940067+00:00", "elapsed_seconds": 77.259, "cost_usd": 0.23013, "known_partial_cost_usd": 0.23013, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 6656, "completion_tokens": 7874, "total_tokens": 14530, "cost": 0.23013, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2817, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.23013, "gateway_cost": 0.23013 }, "finish_reason": "stop", "elapsed_seconds": 77.238, "completed_at": "2026-09-07T02:57:19.937992+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/rent-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-birch", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1695", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-46690", "expected": "BIRCH-REN-46690", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1695" ], "reported_sources": [ "S-R1695" ] }, { "record_id": "R3353", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-48467", "expected": "BIRCH-REN-48467", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3353", "U-R3353-1" ], "reported_sources": [ "S-R3353", "U-R3353-1" ] }, { "record_id": "R2434", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-81113", "expected": "BIRCH-REN-81113", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2434" ], "reported_sources": [ "S-R2434" ] }, { "record_id": "R1930", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-15479", "expected": "BIRCH-REN-15479", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1930" ], "reported_sources": [ "S-R1930" ] }, { "record_id": "R6289", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-REN-11670", "expected": "BIRCH-REN-11670", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6289" ], "reported_sources": [ "S-R6289" ] } ], "decision_checks": [ { "record_id": "R7254", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8772", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2614", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8177", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6289", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3353", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3442", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R5629", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R1930", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1009", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2434", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9966", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7377", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R1695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9256", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5471", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7345", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R3964", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5747", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7562", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-cypress", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:34:06.383Z", "observed_completed_at": "2026-09-07T02:37:46.574Z", "packet_sha256": "318a1e22eb0be8a7d5ea21195823ee09e34e5f791193fcd8f593da099ea7f495", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/cypress_industrial_columbus_rent_comps.xlsx", "sha256": "b95861582ac813ffe21a06ae32f0006d6924d1a00dbacdb5b58e3860a25ccba9", "bytes": 6633 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.5698792500000001, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 132.71, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "bc5a56eb-1887-4512-9d40-1f2d1ad1d1de", "session_id": "fa75e2b1-4813-4f87-850f-e83241a01fb3", "started_at": "2026-09-06T22:35:51.150000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.40421625, 0.165663 ], "total_cost_usd": 0.5698792500000001, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 132.71, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/rent-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-cypress", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R2732", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2732" ], "reported_sources": [ "S-R2732" ] }, { "record_id": "R7377", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7377", "U-R7377-1" ], "reported_sources": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7850" ], "reported_sources": [ "S-R7850" ] }, { "record_id": "R9286", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9286" ], "reported_sources": [ "S-R9286" ] }, { "record_id": "R9636", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9636" ], "reported_sources": [ "S-R9636" ] } ], "decision_checks": [ { "record_id": "R2670", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5936", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R5587", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R3881", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8711", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R7377", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2921", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5588", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3746", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7850", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9636", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2732", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9286", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1369", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7843", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3375", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1188", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7637", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/cypress_industrial_columbus_rent_comps.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Methodology" ], "table_sheet": "Comp Grid", "original_record_order": [ "R2732", "R7377", "R7850", "R9286", "R9636" ], "answer_record_order": [ "R2732", "R7377", "R7850", "R9286", "R9636" ], "field_checks": [ { "record_id": "R2732", "field": "entity_id", "actual": "CYPRESS-REN-54094", "original": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "record_id": "R2732", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R2732", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R2732", "field": "effective_net_rent_psf_year", "actual": 11.2794, "original": 11.2794, "expected": 11.28, "passed": true }, { "record_id": "R2732", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R7377", "field": "entity_id", "actual": "CYPRESS-REN-75613", "original": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "record_id": "R7377", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R7377", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R7377", "field": "effective_net_rent_psf_year", "actual": 16.9256, "original": 16.9256, "expected": 16.93, "passed": true }, { "record_id": "R7377", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R7850", "field": "entity_id", "actual": "CYPRESS-REN-73837", "original": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "record_id": "R7850", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R7850", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R7850", "field": "effective_net_rent_psf_year", "actual": 19.4053, "original": 19.4053, "expected": 19.41, "passed": true }, { "record_id": "R7850", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R9286", "field": "entity_id", "actual": "CYPRESS-REN-55027", "original": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "record_id": "R9286", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R9286", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R9286", "field": "effective_net_rent_psf_year", "actual": 9.468, "original": 9.468, "expected": 9.47, "passed": true }, { "record_id": "R9286", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R9636", "field": "entity_id", "actual": "CYPRESS-REN-82731", "original": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "record_id": "R9636", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R9636", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R9636", "field": "effective_net_rent_psf_year", "actual": 22.1227, "original": 22.1227, "expected": 22.12, "passed": true }, { "record_id": "R9636", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2732", "passed": true, "expected": [ "S-R2732" ], "actual": [ "S-R2732", "S-R5958" ] }, { "record_id": "R7377", "passed": true, "expected": [ "S-R7377", "U-R7377-1" ], "actual": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "passed": true, "expected": [ "S-R7850" ], "actual": [ "S-R7850", "U-R7850-2" ] }, { "record_id": "R9286", "passed": true, "expected": [ "S-R9286" ], "actual": [ "S-R9286", "U-R9286-3" ] }, { "record_id": "R9636", "passed": true, "expected": [ "S-R9636" ], "actual": [ "S-R9636" ] } ], "source_sha256": "b95861582ac813ffe21a06ae32f0006d6924d1a00dbacdb5b58e3860a25ccba9" } }, { "case_id": "rent-comps-cypress", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:56:09.776774+00:00", "completed_at": "2026-09-07T03:02:30.500833+00:00", "elapsed_seconds": 380.723, "cost_usd": 0.35981050000000003, "known_partial_cost_usd": 0.35981050000000003, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 5, "tool_calls": 5, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5643, "completion_tokens": 5153, "total_tokens": 10796, "cost": 0.05858375, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4096, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05858375, "gateway_cost": 0.05858375 }, "finish_reason": "tool_calls", "elapsed_seconds": 56.845, "completed_at": "2026-09-07T02:57:06.637674+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7204, "completion_tokens": 3981, "total_tokens": 11185, "cost": 0.042623, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5504, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3520, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.042623, "gateway_cost": 0.042623 }, "finish_reason": "tool_calls", "elapsed_seconds": 40.445, "completed_at": "2026-09-07T02:57:47.086335+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7855, "completion_tokens": 9148, "total_tokens": 17003, "cost": 0.09323475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7168, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7552, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.09323475, "gateway_cost": 0.09323475 }, "finish_reason": "tool_calls", "elapsed_seconds": 104.145, "completed_at": "2026-09-07T02:59:31.235619+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9480, "completion_tokens": 8210, "total_tokens": 17690, "cost": 0.085166, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7808, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6592, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.085166, "gateway_cost": 0.085166 }, "finish_reason": "tool_calls", "elapsed_seconds": 87.825, "completed_at": "2026-09-07T03:00:59.573650+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 11100, "completion_tokens": 7684, "total_tokens": 18784, "cost": 0.080203, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 9344, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6016, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.080203, "gateway_cost": 0.080203 }, "finish_reason": "tool_calls", "elapsed_seconds": 90.416, "completed_at": "2026-09-07T03:02:30.498042+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "ed117b00fdd2e4d4a9676a959114dca9b3afca659af23b3e1dd67e84e23e59bb", "bytes": 4620 }, { "path": "artifacts/underwriting-v2.xlsx", "sha256": "8066fb1ec750390a2fcdf1e3b20c51437fa5949f84d5eb8bc0dd8db3bd197bef", "bytes": 4613 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/rent-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-cypress", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R2732", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2732" ], "reported_sources": [ "S-R2732" ] }, { "record_id": "R7377", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7377", "U-R7377-1" ], "reported_sources": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.405271557291663, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7850" ], "reported_sources": [ "S-R7850" ] }, { "record_id": "R9286", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9286" ], "reported_sources": [ "S-R9286" ] }, { "record_id": "R9636", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9636" ], "reported_sources": [ "S-R9636" ] } ], "decision_checks": [ { "record_id": "R2670", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5936", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R5587", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R3881", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8711", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R7377", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2921", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5588", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3746", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7850", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9636", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2732", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9286", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1369", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7843", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3375", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1188", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7637", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v2.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comps" ], "table_sheet": "Comps", "original_record_order": [ "R2732", "R7377", "R7850", "R9286", "R9636" ], "answer_record_order": [ "R2732", "R7377", "R7850", "R9286", "R9636" ], "field_checks": [ { "record_id": "R2732", "field": "entity_id", "actual": "CYPRESS-REN-54094", "original": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "record_id": "R2732", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R2732", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R2732", "field": "effective_net_rent_psf_year", "actual": 11.279375, "original": 11.279375, "expected": 11.279375, "passed": true }, { "record_id": "R2732", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R7377", "field": "entity_id", "actual": "CYPRESS-REN-75613", "original": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "record_id": "R7377", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R7377", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R7377", "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "original": 16.92561265625, "expected": 16.92561265625, "passed": true }, { "record_id": "R7377", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R7850", "field": "entity_id", "actual": "CYPRESS-REN-73837", "original": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "record_id": "R7850", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R7850", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R7850", "field": "effective_net_rent_psf_year", "actual": 19.405271557291663, "original": 19.405271557291663, "expected": 19.405271557291663, "passed": true }, { "record_id": "R7850", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R9286", "field": "entity_id", "actual": "CYPRESS-REN-55027", "original": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "record_id": "R9286", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R9286", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R9286", "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "original": 9.468024999999992, "expected": 9.468024999999992, "passed": true }, { "record_id": "R9286", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R9636", "field": "entity_id", "actual": "CYPRESS-REN-82731", "original": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "record_id": "R9636", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R9636", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R9636", "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "original": 22.122719531249995, "expected": 22.122719531249995, "passed": true }, { "record_id": "R9636", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2732", "passed": true, "expected": [ "S-R2732" ], "actual": [ "S-R2732" ] }, { "record_id": "R7377", "passed": true, "expected": [ "S-R7377", "U-R7377-1" ], "actual": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "passed": true, "expected": [ "S-R7850" ], "actual": [ "S-R7850" ] }, { "record_id": "R9286", "passed": true, "expected": [ "S-R9286" ], "actual": [ "S-R9286" ] }, { "record_id": "R9636", "passed": true, "expected": [ "S-R9636" ], "actual": [ "S-R9636" ] } ], "source_sha256": "8066fb1ec750390a2fcdf1e3b20c51437fa5949f84d5eb8bc0dd8db3bd197bef" } }, { "case_id": "rent-comps-cypress", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:57:19.941171+00:00", "completed_at": "2026-09-07T02:59:39.262956+00:00", "elapsed_seconds": 139.321, "cost_usd": 0.511805, "known_partial_cost_usd": 0.511805, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8971, "completion_tokens": 2777, "total_tokens": 11748, "cost": 0.11428, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2261, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11428, "gateway_cost": 0.11428 }, "finish_reason": "tool_calls", "elapsed_seconds": 32.191, "completed_at": "2026-09-07T02:57:52.153126+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9600, "completion_tokens": 6659, "total_tokens": 16259, "cost": 0.214475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1823, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.214475, "gateway_cost": 0.214475 }, "finish_reason": "tool_calls", "elapsed_seconds": 62.334, "completed_at": "2026-09-07T02:58:54.490353+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 15110, "completion_tokens": 4300, "total_tokens": 19410, "cost": 0.18305, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 209, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.18305, "gateway_cost": 0.18305 }, "finish_reason": "stop", "elapsed_seconds": 44.238, "completed_at": "2026-09-07T02:59:39.261249+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "7a76e921ed88141a16099d61d0fdfc15d8fee8a7d4cee5fbf27519caa28c4429", "bytes": 5883 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/rent-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-cypress", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R2732", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2732" ], "reported_sources": [ "S-R2732" ] }, { "record_id": "R7377", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7377", "U-R7377-1" ], "reported_sources": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7850" ], "reported_sources": [ "S-R7850" ] }, { "record_id": "R9286", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9286" ], "reported_sources": [ "S-R9286" ] }, { "record_id": "R9636", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9636" ], "reported_sources": [ "S-R9636" ] } ], "decision_checks": [ { "record_id": "R2670", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5936", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R5587", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R3881", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8711", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R7377", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2921", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5588", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3746", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7850", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9636", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2732", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9286", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1369", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7843", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3375", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1188", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7637", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R2732", "R7377", "R7850", "R9286", "R9636" ], "answer_record_order": [ "R2732", "R7377", "R7850", "R9286", "R9636" ], "field_checks": [ { "record_id": "R2732", "field": "entity_id", "actual": "CYPRESS-REN-54094", "original": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "record_id": "R2732", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R2732", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R2732", "field": "effective_net_rent_psf_year", "actual": 11.279374999999998, "original": 11.279374999999998, "expected": 11.28, "passed": true }, { "record_id": "R2732", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R7377", "field": "entity_id", "actual": "CYPRESS-REN-75613", "original": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "record_id": "R7377", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R7377", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R7377", "field": "effective_net_rent_psf_year", "actual": 16.925612656249996, "original": 16.925612656249996, "expected": 16.93, "passed": true }, { "record_id": "R7377", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R7850", "field": "entity_id", "actual": "CYPRESS-REN-73837", "original": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "record_id": "R7850", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R7850", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R7850", "field": "effective_net_rent_psf_year", "actual": 19.40527155729166, "original": 19.40527155729166, "expected": 19.41, "passed": true }, { "record_id": "R7850", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R9286", "field": "entity_id", "actual": "CYPRESS-REN-55027", "original": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "record_id": "R9286", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R9286", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R9286", "field": "effective_net_rent_psf_year", "actual": 9.468024999999997, "original": 9.468024999999997, "expected": 9.47, "passed": true }, { "record_id": "R9286", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R9636", "field": "entity_id", "actual": "CYPRESS-REN-82731", "original": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "record_id": "R9636", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R9636", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R9636", "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "original": 22.122719531249995, "expected": 22.12, "passed": true }, { "record_id": "R9636", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2732", "passed": true, "expected": [ "S-R2732" ], "actual": [ "S-R2732", "S-R5958" ] }, { "record_id": "R7377", "passed": true, "expected": [ "S-R7377", "U-R7377-1" ], "actual": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "passed": true, "expected": [ "S-R7850" ], "actual": [ "S-R7850", "U-R7850-2" ] }, { "record_id": "R9286", "passed": true, "expected": [ "S-R9286" ], "actual": [ "S-R9286", "U-R9286-3" ] }, { "record_id": "R9636", "passed": true, "expected": [ "S-R9636" ], "actual": [ "S-R9636" ] } ], "source_sha256": "7a76e921ed88141a16099d61d0fdfc15d8fee8a7d4cee5fbf27519caa28c4429" } }, { "case_id": "rent-comps-cypress", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:58:36.758757+00:00", "completed_at": "2026-09-07T03:00:15.277195+00:00", "elapsed_seconds": 98.518, "cost_usd": 0.14908125, "known_partial_cost_usd": 0.14908125, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4897, "completion_tokens": 14296, "total_tokens": 19193, "cost": 0.14908125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 11904, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.14908125, "gateway_cost": 0.14908125 }, "finish_reason": "stop", "elapsed_seconds": 98.5, "completed_at": "2026-09-07T03:00:15.275950+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/rent-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-cypress", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R2732", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2732" ], "reported_sources": [ "S-R2732" ] }, { "record_id": "R7377", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7377", "U-R7377-1" ], "reported_sources": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.405271557285, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7850" ], "reported_sources": [ "S-R7850" ] }, { "record_id": "R9286", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468025, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9286" ], "reported_sources": [ "S-R9286" ] }, { "record_id": "R9636", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12271953125, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9636" ], "reported_sources": [ "S-R9636" ] } ], "decision_checks": [ { "record_id": "R2670", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5936", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R5587", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R3881", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8711", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R7377", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2921", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5588", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3746", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7850", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9636", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2732", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9286", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1369", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7843", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3375", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1188", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7637", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-cypress", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:59:39.264770+00:00", "completed_at": "2026-09-07T03:00:58.965824+00:00", "elapsed_seconds": 79.701, "cost_usd": 0.226705, "known_partial_cost_usd": 0.226705, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 6686, "completion_tokens": 7731, "total_tokens": 14417, "cost": 0.226705, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3055, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.226705, "gateway_cost": 0.226705 }, "finish_reason": "stop", "elapsed_seconds": 79.682, "completed_at": "2026-09-07T03:00:58.964394+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/rent-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-cypress", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R2732", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-54094", "expected": "CYPRESS-REN-54094", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2732" ], "reported_sources": [ "S-R2732", "S-R5958" ] }, { "record_id": "R7377", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-75613", "expected": "CYPRESS-REN-75613", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7377", "U-R7377-1" ], "reported_sources": [ "S-R7377", "U-R7377-1" ] }, { "record_id": "R7850", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-73837", "expected": "CYPRESS-REN-73837", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7850" ], "reported_sources": [ "S-R7850" ] }, { "record_id": "R9286", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-55027", "expected": "CYPRESS-REN-55027", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9286" ], "reported_sources": [ "S-R9286" ] }, { "record_id": "R9636", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-REN-82731", "expected": "CYPRESS-REN-82731", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9636" ], "reported_sources": [ "S-R9636" ] } ], "decision_checks": [ { "record_id": "R2670", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5936", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R5587", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R3881", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R5958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8711", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R7377", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2921", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5588", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3746", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7850", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9636", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2732", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9286", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1369", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7843", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3375", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1188", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7637", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-dogwood", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:34:10.283Z", "observed_completed_at": "2026-09-07T02:37:46.755Z", "packet_sha256": "e981441f5c4fa8861e0a359614ece51a5240041d824fa9f21d79deb249fa4f11", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/dogwood_rent_comp_grid.xlsx", "sha256": "7e787187e75814ddc1950d8f59acc356e06dfd81c7f3a7e9eb16c1c67e9482b1", "bytes": 7406 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.60725825, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 130.01, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "636d9dee-0840-4129-9319-69a00fc7d3f9", "session_id": "5aea640a-7739-48ff-be60-da11bc279f49", "started_at": "2026-09-06T22:35:53.296000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.41746875, 0.1897895 ], "total_cost_usd": 0.60725825, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 130.01, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/rent-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-dogwood", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R8720", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8720" ], "reported_sources": [ "S-R8720" ] }, { "record_id": "R3534", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3534", "U-R3534-1" ], "reported_sources": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6645" ], "reported_sources": [ "S-R6645" ] }, { "record_id": "R6812", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6812" ], "reported_sources": [ "S-R6812" ] }, { "record_id": "R1551", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1551" ], "reported_sources": [ "S-R1551" ] } ], "decision_checks": [ { "record_id": "R8720", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8926", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R9647", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9438", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R6157", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4285", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9947", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6717", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9661", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R6613", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6645", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9315", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R7718", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1330", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1551", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6660", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9145", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4220", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6812", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3534", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/dogwood_rent_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions" ], "table_sheet": "Comp Grid", "original_record_order": [ "R8720", "R3534", "R6645", "R6812", "R1551" ], "answer_record_order": [ "R8720", "R3534", "R6645", "R6812", "R1551" ], "field_checks": [ { "record_id": "R8720", "field": "entity_id", "actual": "DOGWOOD-REN-60901", "original": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "record_id": "R8720", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R8720", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R8720", "field": "effective_net_rent_psf_year", "actual": 11.28, "original": 11.28, "expected": 11.28, "passed": true }, { "record_id": "R8720", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R3534", "field": "entity_id", "actual": "DOGWOOD-REN-79987", "original": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "record_id": "R3534", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R3534", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R3534", "field": "effective_net_rent_psf_year", "actual": 16.93, "original": 16.93, "expected": 16.93, "passed": true }, { "record_id": "R3534", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R6645", "field": "entity_id", "actual": "DOGWOOD-REN-64895", "original": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "record_id": "R6645", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R6645", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R6645", "field": "effective_net_rent_psf_year", "actual": 19.41, "original": 19.41, "expected": 19.41, "passed": true }, { "record_id": "R6645", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R6812", "field": "entity_id", "actual": "DOGWOOD-REN-50280", "original": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "record_id": "R6812", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R6812", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R6812", "field": "effective_net_rent_psf_year", "actual": 9.47, "original": 9.47, "expected": 9.47, "passed": true }, { "record_id": "R6812", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R1551", "field": "entity_id", "actual": "DOGWOOD-REN-99067", "original": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "record_id": "R1551", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1551", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R1551", "field": "effective_net_rent_psf_year", "actual": 22.12, "original": 22.12, "expected": 22.12, "passed": true }, { "record_id": "R1551", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R8720", "passed": true, "expected": [ "S-R8720" ], "actual": [ "S-R8720" ] }, { "record_id": "R3534", "passed": true, "expected": [ "S-R3534", "U-R3534-1" ], "actual": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "passed": true, "expected": [ "S-R6645" ], "actual": [ "S-R6645", "U-R6645-2" ] }, { "record_id": "R6812", "passed": true, "expected": [ "S-R6812" ], "actual": [ "S-R6812", "U-R6812-3" ] }, { "record_id": "R1551", "passed": true, "expected": [ "S-R1551" ], "actual": [ "S-R1551" ] } ], "source_sha256": "7e787187e75814ddc1950d8f59acc356e06dfd81c7f3a7e9eb16c1c67e9482b1" } }, { "case_id": "rent-comps-dogwood", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:00:15.277905+00:00", "completed_at": "2026-09-07T03:04:58.587192+00:00", "elapsed_seconds": 283.31, "cost_usd": 0.27104775, "known_partial_cost_usd": 0.27104775, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 4, "tool_calls": 4, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5647, "completion_tokens": 2851, "total_tokens": 8498, "cost": 0.03556875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2560, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.03556875, "gateway_cost": 0.03556875 }, "finish_reason": "tool_calls", "elapsed_seconds": 35.573, "completed_at": "2026-09-07T03:00:50.866661+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6059, "completion_tokens": 4911, "total_tokens": 10970, "cost": 0.05049175, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5504, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3904, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05049175, "gateway_cost": 0.05049175 }, "finish_reason": "tool_calls", "elapsed_seconds": 56.096, "completed_at": "2026-09-07T03:01:46.965688+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7615, "completion_tokens": 9912, "total_tokens": 17527, "cost": 0.10187075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6016, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 8384, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10187075, "gateway_cost": 0.10187075 }, "finish_reason": "tool_calls", "elapsed_seconds": 105.667, "completed_at": "2026-09-07T03:03:32.636268+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9130, "completion_tokens": 8020, "total_tokens": 17150, "cost": 0.0831165, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7552, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6272, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0831165, "gateway_cost": 0.0831165 }, "finish_reason": "tool_calls", "elapsed_seconds": 85.427, "completed_at": "2026-09-07T03:04:58.584466+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "b14852bbcd704bf28f074d1a85bf213146d38a650078e17fa822b63128d5b3b5", "bytes": 4523 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/rent-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-dogwood", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R8720", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8720" ], "reported_sources": [ "S-R8720" ] }, { "record_id": "R3534", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3534", "U-R3534-1" ], "reported_sources": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6645" ], "reported_sources": [ "S-R6645" ] }, { "record_id": "R6812", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6812" ], "reported_sources": [ "S-R6812" ] }, { "record_id": "R1551", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.122719531249995, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1551" ], "reported_sources": [ "S-R1551" ] } ], "decision_checks": [ { "record_id": "R8720", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8926", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R9647", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9438", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R6157", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4285", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9947", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6717", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9661", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R6613", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6645", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9315", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R7718", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1330", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1551", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6660", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9145", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4220", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6812", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3534", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R8720", "R3534", "R6645", "R6812", "R1551" ], "answer_record_order": [ "R8720", "R3534", "R6645", "R6812", "R1551" ], "field_checks": [ { "record_id": "R8720", "field": "entity_id", "actual": "DOGWOOD-REN-60901", "original": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "record_id": "R8720", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R8720", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21, "passed": true }, { "record_id": "R8720", "field": "effective_net_rent_psf_year", "actual": 11.279375, "original": 11.279375, "expected": 11.279375, "passed": true }, { "record_id": "R8720", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R3534", "field": "entity_id", "actual": "DOGWOOD-REN-79987", "original": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "record_id": "R3534", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R3534", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R3534", "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "original": 16.92561265625, "expected": 16.92561265625, "passed": true }, { "record_id": "R3534", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R6645", "field": "entity_id", "actual": "DOGWOOD-REN-64895", "original": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "record_id": "R6645", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R6645", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R6645", "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "original": 19.405271557291666, "expected": 19.405271557291666, "passed": true }, { "record_id": "R6645", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R6812", "field": "entity_id", "actual": "DOGWOOD-REN-50280", "original": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "record_id": "R6812", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R6812", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R6812", "field": "effective_net_rent_psf_year", "actual": 9.468024999999992, "original": 9.468024999999992, "expected": 9.468024999999992, "passed": true }, { "record_id": "R6812", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R1551", "field": "entity_id", "actual": "DOGWOOD-REN-99067", "original": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "record_id": "R1551", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1551", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R1551", "field": "effective_net_rent_psf_year", "actual": 22.12271953125, "original": 22.12271953125, "expected": 22.122719531249995, "passed": true }, { "record_id": "R1551", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R8720", "passed": true, "expected": [ "S-R8720" ], "actual": [ "S-R8720" ] }, { "record_id": "R3534", "passed": true, "expected": [ "S-R3534", "U-R3534-1" ], "actual": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "passed": true, "expected": [ "S-R6645" ], "actual": [ "S-R6645" ] }, { "record_id": "R6812", "passed": true, "expected": [ "S-R6812" ], "actual": [ "S-R6812" ] }, { "record_id": "R1551", "passed": true, "expected": [ "S-R1551" ], "actual": [ "S-R1551" ] } ], "source_sha256": "b14852bbcd704bf28f074d1a85bf213146d38a650078e17fa822b63128d5b3b5" } }, { "case_id": "rent-comps-dogwood", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:00:58.966824+00:00", "completed_at": "2026-09-07T03:03:43.716940+00:00", "elapsed_seconds": 164.751, "cost_usd": 0.596995, "known_partial_cost_usd": 0.596995, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 4, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9024, "completion_tokens": 2467, "total_tokens": 11491, "cost": 0.106795, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2050, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.106795, "gateway_cost": 0.106795 }, "finish_reason": "tool_calls", "elapsed_seconds": 27.665, "completed_at": "2026-09-07T03:01:26.650475+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9539, "completion_tokens": 1859, "total_tokens": 11398, "cost": 0.09417, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1774, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.09417, "gateway_cost": 0.09417 }, "finish_reason": "tool_calls", "elapsed_seconds": 24.686, "completed_at": "2026-09-07T03:01:51.340006+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9682, "completion_tokens": 6351, "total_tokens": 16033, "cost": 0.207185, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2462, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.207185, "gateway_cost": 0.207185 }, "finish_reason": "tool_calls", "elapsed_seconds": 63.488, "completed_at": "2026-09-07T03:02:54.829189+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13804, "completion_tokens": 4793, "total_tokens": 18597, "cost": 0.188845, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 502, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.188845, "gateway_cost": 0.188845 }, "finish_reason": "stop", "elapsed_seconds": 48.358, "completed_at": "2026-09-07T03:03:43.715012+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "847c8f4a2de9db668508a13a68433413fc88726864e19e461c657c268c5cea65", "bytes": 5463 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/rent-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-dogwood", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R8720", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8720" ], "reported_sources": [ "S-R8720", "S-R6660" ] }, { "record_id": "R3534", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3534", "U-R3534-1" ], "reported_sources": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6645" ], "reported_sources": [ "S-R6645" ] }, { "record_id": "R6812", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6812" ], "reported_sources": [ "S-R6812" ] }, { "record_id": "R1551", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1551" ], "reported_sources": [ "S-R1551" ] } ], "decision_checks": [ { "record_id": "R8720", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8926", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R9647", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9438", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R6157", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4285", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9947", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6717", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9661", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R6613", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6645", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9315", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R7718", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1330", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1551", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6660", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9145", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4220", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6812", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3534", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R8720", "R3534", "R6645", "R6812", "R1551" ], "answer_record_order": [ "R8720", "R3534", "R6645", "R6812", "R1551" ], "field_checks": [ { "record_id": "R8720", "field": "entity_id", "actual": "DOGWOOD-REN-60901", "original": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "record_id": "R8720", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R8720", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R8720", "field": "effective_net_rent_psf_year", "actual": 11.279374999999902, "original": 11.279374999999902, "expected": 11.28, "passed": true }, { "record_id": "R8720", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R3534", "field": "entity_id", "actual": "DOGWOOD-REN-79987", "original": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "record_id": "R3534", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R3534", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R3534", "field": "effective_net_rent_psf_year", "actual": 16.925612656249896, "original": 16.925612656249896, "expected": 16.93, "passed": true }, { "record_id": "R3534", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R6645", "field": "entity_id", "actual": "DOGWOOD-REN-64895", "original": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "record_id": "R6645", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R6645", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R6645", "field": "effective_net_rent_psf_year", "actual": 19.405271557291563, "original": 19.405271557291563, "expected": 19.41, "passed": true }, { "record_id": "R6645", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R6812", "field": "entity_id", "actual": "DOGWOOD-REN-50280", "original": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "record_id": "R6812", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R6812", "field": "annual_face_rent_psf", "actual": 24.12, "original": 24.12, "expected": 24.12, "passed": true }, { "record_id": "R6812", "field": "effective_net_rent_psf_year", "actual": 9.468024999999887, "original": 9.468024999999887, "expected": 9.47, "passed": true }, { "record_id": "R6812", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R1551", "field": "entity_id", "actual": "DOGWOOD-REN-99067", "original": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "record_id": "R1551", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1551", "field": "annual_face_rent_psf", "actual": 24.2, "original": 24.2, "expected": 24.2, "passed": true }, { "record_id": "R1551", "field": "effective_net_rent_psf_year", "actual": 22.12271953124989, "original": 22.12271953124989, "expected": 22.12, "passed": true }, { "record_id": "R1551", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R8720", "passed": true, "expected": [ "S-R6660", "S-R8720" ], "actual": [ "S-R6660", "S-R8720" ] }, { "record_id": "R3534", "passed": true, "expected": [ "S-R3534", "U-R3534-1" ], "actual": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "passed": true, "expected": [ "S-R6645" ], "actual": [ "S-R6645", "U-R6645-2" ] }, { "record_id": "R6812", "passed": true, "expected": [ "S-R6812" ], "actual": [ "S-R6812", "U-R6812-3" ] }, { "record_id": "R1551", "passed": true, "expected": [ "S-R1551" ], "actual": [ "S-R1551" ] } ], "source_sha256": "847c8f4a2de9db668508a13a68433413fc88726864e19e461c657c268c5cea65" } }, { "case_id": "rent-comps-dogwood", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:02:30.502765+00:00", "completed_at": "2026-09-07T03:04:44.371613+00:00", "elapsed_seconds": 133.87, "cost_usd": 0.13247625, "known_partial_cost_usd": 0.13247625, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4901, "completion_tokens": 12635, "total_tokens": 17536, "cost": 0.13247625, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 10368, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.13247625, "gateway_cost": 0.13247625 }, "finish_reason": "stop", "elapsed_seconds": 133.85, "completed_at": "2026-09-07T03:04:44.370136+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/rent-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-dogwood", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R8720", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8720" ], "reported_sources": [ "S-R8720" ] }, { "record_id": "R3534", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3534", "U-R3534-1" ], "reported_sources": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6645" ], "reported_sources": [ "S-R6645" ] }, { "record_id": "R6812", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6812" ], "reported_sources": [ "S-R6812" ] }, { "record_id": "R1551", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1551" ], "reported_sources": [ "S-R1551" ] } ], "decision_checks": [ { "record_id": "R8720", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8926", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R9647", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9438", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R6157", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4285", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9947", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6717", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9661", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R6613", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6645", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9315", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R7718", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1330", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1551", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6660", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9145", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4220", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6812", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3534", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-dogwood", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:03:43.718087+00:00", "completed_at": "2026-09-07T03:04:58.692508+00:00", "elapsed_seconds": 74.974, "cost_usd": 0.21937, "known_partial_cost_usd": 0.21937, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 6739, "completion_tokens": 7427, "total_tokens": 14166, "cost": 0.21937, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2561, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.21937, "gateway_cost": 0.21937 }, "finish_reason": "stop", "elapsed_seconds": 74.957, "completed_at": "2026-09-07T03:04:58.690193+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/rent-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "rent-comps-dogwood", "task": "rent_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R8720", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-60901", "expected": "DOGWOOD-REN-60901", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8720" ], "reported_sources": [ "S-R8720", "S-R6660" ] }, { "record_id": "R3534", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-79987", "expected": "DOGWOOD-REN-79987", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3534", "U-R3534-1" ], "reported_sources": [ "S-R3534", "U-R3534-1" ] }, { "record_id": "R6645", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-64895", "expected": "DOGWOOD-REN-64895", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6645" ], "reported_sources": [ "S-R6645" ] }, { "record_id": "R6812", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-50280", "expected": "DOGWOOD-REN-50280", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.12, "expected": 24.119999999999997, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 9.47, "expected": 9.468025, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6812" ], "reported_sources": [ "S-R6812" ] }, { "record_id": "R1551", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-REN-99067", "expected": "DOGWOOD-REN-99067", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "annual_face_rent_psf", "actual": 24.2, "expected": 24.2, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 22.12, "expected": 22.122719531250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1551" ], "reported_sources": [ "S-R1551" ] } ], "decision_checks": [ { "record_id": "R8720", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8926", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R9647", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9438", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R6157", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R4285", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9947", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6717", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9661", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R6613", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6645", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9315", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R7718", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1330", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R1551", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6660", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9145", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R4220", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6812", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3534", "actual": true, "expected": true, "passed": true, "reasons": [] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-elm", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:39:02.408Z", "observed_completed_at": "2026-09-07T02:44:55.056Z", "packet_sha256": "331e29405a4a7bb44254d0d2445623a1a1a7aee70a12743e9f36ac183f8bfd01", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/rent_comps_raleigh_retail.xlsx", "sha256": "bc1514e04fabdefa096aa15af6a0a5c5bcbce7c425ca5bc0e7faf27b4b570345", "bytes": 8322 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.603819, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 135.5, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "162125d6-5330-4a40-a85c-cee98d8945aa", "session_id": "d1660119-4c9e-4a5d-94a2-ac8ea64b0cb5", "started_at": "2026-09-06T22:40:46.216000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.43014825, 0.17367075 ], "total_cost_usd": 0.603819, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 135.5, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/rent-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "rent-comps-elm", "task": "rent_comps", "target_count": 3, "available_unique": 3, "returned_count": 4, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.75, "correct_abstention": null, "field_accuracy": { "passed": 20, "total": 20 }, "evidence_accuracy": { "passed": 4, "total": 4 }, "candidate_accuracy": { "passed": 18, "total": 20 }, "eligible_recall": { "passed": 3, "total": 4 }, "ineligible_rejection": { "passed": 15, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [ { "record_id": "R4617", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R5361", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.2794, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5361" ], "reported_sources": [ "S-R5361" ] }, { "record_id": "R8528", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.9256, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8528", "U-R8528-1" ], "reported_sources": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.4053, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4908" ], "reported_sources": [ "S-R4908" ] }, { "record_id": "R4617", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-77062", "expected": "ELM-REN-77062", "passed": true }, { "field": "event_date", "actual": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "field": "annual_face_rent_psf", "actual": 35.88, "expected": 35.88, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 34.1214, "expected": 34.12141342812504, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4617" ], "reported_sources": [ "S-R4617" ] } ], "decision_checks": [ { "record_id": "R8323", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R5055", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5909", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6803", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R4908", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7763", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8668", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6572", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R3616", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R3778", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1184", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R8329", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5806", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2825", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9355", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5361", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8578", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R8528", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6603", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R4617", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/rent_comps_raleigh_retail.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions", "Summary" ], "table_sheet": "Comp Grid", "original_record_order": [ "R5361", "R8528", "R4908", "R4617" ], "answer_record_order": [ "R5361", "R8528", "R4908", "R4617" ], "field_checks": [ { "record_id": "R5361", "field": "entity_id", "actual": "ELM-REN-25776", "original": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "record_id": "R5361", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5361", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R5361", "field": "effective_net_rent_psf_year", "actual": 11.2794, "original": 11.2794, "expected": 11.2794, "passed": true }, { "record_id": "R5361", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R8528", "field": "entity_id", "actual": "ELM-REN-95631", "original": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "record_id": "R8528", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8528", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R8528", "field": "effective_net_rent_psf_year", "actual": 16.9256, "original": 16.9256, "expected": 16.9256, "passed": true }, { "record_id": "R8528", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R4908", "field": "entity_id", "actual": "ELM-REN-51407", "original": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "record_id": "R4908", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R4908", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R4908", "field": "effective_net_rent_psf_year", "actual": 19.4053, "original": 19.4053, "expected": 19.4053, "passed": true }, { "record_id": "R4908", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true }, { "record_id": "R4617", "field": "entity_id", "actual": "ELM-REN-77062", "original": "ELM-REN-77062", "expected": "ELM-REN-77062", "passed": true }, { "record_id": "R4617", "field": "event_date", "actual": "2026-07-22", "original": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "record_id": "R4617", "field": "annual_face_rent_psf", "actual": 35.88, "original": 35.88, "expected": 35.88, "passed": true }, { "record_id": "R4617", "field": "effective_net_rent_psf_year", "actual": 34.1214, "original": 34.1214, "expected": 34.1214, "passed": true }, { "record_id": "R4617", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5361", "passed": true, "expected": [ "S-R5361" ], "actual": [ "S-R5361" ] }, { "record_id": "R8528", "passed": true, "expected": [ "S-R8528", "U-R8528-1" ], "actual": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "passed": true, "expected": [ "S-R4908" ], "actual": [ "S-R4908" ] }, { "record_id": "R4617", "passed": true, "expected": [ "S-R4617" ], "actual": [ "S-R4617" ] } ], "source_sha256": "bc1514e04fabdefa096aa15af6a0a5c5bcbce7c425ca5bc0e7faf27b4b570345" } }, { "case_id": "rent-comps-elm", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:04:44.372454+00:00", "completed_at": "2026-09-07T03:08:13.736301+00:00", "elapsed_seconds": 209.363, "cost_usd": 0.18845425000000002, "known_partial_cost_usd": 0.18845425000000002, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 5, "tool_calls": 5, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5920, "completion_tokens": 3231, "total_tokens": 9151, "cost": 0.03971, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3072, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.03971, "gateway_cost": 0.03971 }, "finish_reason": "tool_calls", "elapsed_seconds": 61.759, "completed_at": "2026-09-07T03:05:46.149884+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6080, "completion_tokens": 3995, "total_tokens": 10075, "cost": 0.040926, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5888, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 3456, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.040926, "gateway_cost": 0.040926 }, "finish_reason": "tool_calls", "elapsed_seconds": 47.183, "completed_at": "2026-09-07T03:06:33.338966+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6904, "completion_tokens": 8655, "total_tokens": 15559, "cost": 0.088412, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6016, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7616, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.088412, "gateway_cost": 0.088412 }, "finish_reason": "tool_calls", "elapsed_seconds": 85.785, "completed_at": "2026-09-07T03:07:59.126756+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7982, "completion_tokens": 169, "total_tokens": 8151, "cost": 0.0040355, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6784, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 0, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0040355, "gateway_cost": 0.0040355 }, "finish_reason": "tool_calls", "elapsed_seconds": 1.788, "completed_at": "2026-09-07T03:08:01.427028+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8175, "completion_tokens": 1408, "total_tokens": 9583, "cost": 0.01537075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7936, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 0, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.01537075, "gateway_cost": 0.01537075 }, "finish_reason": "tool_calls", "elapsed_seconds": 12.304, "completed_at": "2026-09-07T03:08:13.733885+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "5fbeba4b6a4dfae2865fc57a8b4efab1cae27acbf1f87d55aa176ebf4c94dfb3", "bytes": 301 }, { "path": "artifacts/underwriting-v1.xlsx", "sha256": "6cf81f70d4a55c26027acaf63419a7ed255fadd6119ff3df73946ba03c165e44", "bytes": 4325 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/rent-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "rent-comps-elm", "task": "rent_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 17, "total": 20 }, "eligible_recall": { "passed": 3, "total": 4 }, "ineligible_rejection": { "passed": 14, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R5361", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.279375, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5361" ], "reported_sources": [ "S-R5361" ] }, { "record_id": "R8528", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8528", "U-R8528-1" ], "reported_sources": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4908" ], "reported_sources": [ "S-R4908" ] } ], "decision_checks": [ { "record_id": "R8323", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R5055", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5909", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6803", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R4908", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7763", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8668", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6572", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R3616", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R3778", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1184", "actual": true, "expected": false, "passed": false, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R8329", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5806", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2825", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9355", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5361", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8578", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R8528", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6603", "actual": true, "expected": false, "passed": false, "reasons": [ "Unknown free_months" ] }, { "record_id": "R4617", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comps" ], "table_sheet": "Comps", "original_record_order": [ "R5361", "R8528", "R4908" ], "answer_record_order": [ "R5361", "R8528", "R4908" ], "field_checks": [ { "record_id": "R5361", "field": "entity_id", "actual": "ELM-REN-25776", "original": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "record_id": "R5361", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5361", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21, "passed": true }, { "record_id": "R5361", "field": "effective_net_rent_psf_year", "actual": 11.279375, "original": 11.279375, "expected": 11.279375, "passed": true }, { "record_id": "R5361", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R8528", "field": "entity_id", "actual": "ELM-REN-95631", "original": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "record_id": "R8528", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8528", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R8528", "field": "effective_net_rent_psf_year", "actual": 16.92561265625, "original": 16.92561265625, "expected": 16.92561265625, "passed": true }, { "record_id": "R8528", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R4908", "field": "entity_id", "actual": "ELM-REN-51407", "original": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "record_id": "R4908", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R4908", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R4908", "field": "effective_net_rent_psf_year", "actual": 19.405271557291666, "original": 19.405271557291666, "expected": 19.405271557291666, "passed": true }, { "record_id": "R4908", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5361", "passed": true, "expected": [ "S-R5361" ], "actual": [ "S-R5361" ] }, { "record_id": "R8528", "passed": true, "expected": [ "S-R8528", "U-R8528-1" ], "actual": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "passed": true, "expected": [ "S-R4908" ], "actual": [ "S-R4908" ] } ], "source_sha256": "6cf81f70d4a55c26027acaf63419a7ed255fadd6119ff3df73946ba03c165e44" } }, { "case_id": "rent-comps-elm", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:04:58.588223+00:00", "completed_at": "2026-09-07T03:06:47.137809+00:00", "elapsed_seconds": 108.549, "cost_usd": 0.41118499999999997, "known_partial_cost_usd": 0.41118499999999997, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9299, "completion_tokens": 1985, "total_tokens": 11284, "cost": 0.09612, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1604, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.09612, "gateway_cost": 0.09612 }, "finish_reason": "tool_calls", "elapsed_seconds": 22.989, "completed_at": "2026-09-07T03:05:21.594678+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9859, "completion_tokens": 3565, "total_tokens": 13424, "cost": 0.13842, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1383, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.13842, "gateway_cost": 0.13842 }, "finish_reason": "tool_calls", "elapsed_seconds": 37.44, "completed_at": "2026-09-07T03:05:59.037681+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12429, "completion_tokens": 4580, "total_tokens": 17009, "cost": 0.176645, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 865, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.176645, "gateway_cost": 0.176645 }, "finish_reason": "stop", "elapsed_seconds": 47.58, "completed_at": "2026-09-07T03:06:47.136931+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "e402dc41bb339446d4b926996a31317284c0ffe3000a6d8341d1122736354604", "bytes": 4891 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/rent-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "rent-comps-elm", "task": "rent_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R5361", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5361" ], "reported_sources": [ "S-R5361" ] }, { "record_id": "R8528", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8528", "U-R8528-1" ], "reported_sources": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4908" ], "reported_sources": [ "S-R4908" ] } ], "decision_checks": [ { "record_id": "R8323", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R5055", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5909", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6803", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R4908", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7763", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8668", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6572", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3616", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R3778", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1184", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R8329", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5806", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2825", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9355", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5361", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8578", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R8528", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6603", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R4617", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R5361", "R8528", "R4908" ], "answer_record_order": [ "R5361", "R8528", "R4908" ], "field_checks": [ { "record_id": "R5361", "field": "entity_id", "actual": "ELM-REN-25776", "original": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "record_id": "R5361", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5361", "field": "annual_face_rent_psf", "actual": 21, "original": 21, "expected": 21.0, "passed": true }, { "record_id": "R5361", "field": "effective_net_rent_psf_year", "actual": 11.279374999999998, "original": 11.279374999999998, "expected": 11.28, "passed": true }, { "record_id": "R5361", "field": "term_months", "actual": 36, "original": 36, "expected": 36, "passed": true }, { "record_id": "R8528", "field": "entity_id", "actual": "ELM-REN-95631", "original": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "record_id": "R8528", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8528", "field": "annual_face_rent_psf", "actual": 22.44, "original": 22.44, "expected": 22.44, "passed": true }, { "record_id": "R8528", "field": "effective_net_rent_psf_year", "actual": 16.925612656249996, "original": 16.925612656249996, "expected": 16.93, "passed": true }, { "record_id": "R8528", "field": "term_months", "actual": 48, "original": 48, "expected": 48, "passed": true }, { "record_id": "R4908", "field": "entity_id", "actual": "ELM-REN-51407", "original": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "record_id": "R4908", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R4908", "field": "annual_face_rent_psf", "actual": 22.6, "original": 22.6, "expected": 22.6, "passed": true }, { "record_id": "R4908", "field": "effective_net_rent_psf_year", "actual": 19.40527155729166, "original": 19.40527155729166, "expected": 19.41, "passed": true }, { "record_id": "R4908", "field": "term_months", "actual": 60, "original": 60, "expected": 60, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5361", "passed": true, "expected": [ "S-R5361" ], "actual": [ "S-R5361", "S-R6572" ] }, { "record_id": "R8528", "passed": true, "expected": [ "S-R8528", "U-R8528-1" ], "actual": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "passed": true, "expected": [ "S-R4908" ], "actual": [ "S-R4908", "U-R4908-2" ] } ], "source_sha256": "e402dc41bb339446d4b926996a31317284c0ffe3000a6d8341d1122736354604" } }, { "case_id": "rent-comps-elm", "system": "openai/gpt-5", "track": "direct", "status": "infrastructure_error", "started_at": "2026-09-07T03:04:58.693825+00:00", "completed_at": "2026-09-07T03:15:00.819055+00:00", "elapsed_seconds": 602.123, "cost_usd": null, "known_partial_cost_usd": 0, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [], "model_calls": 2, "tool_calls": 0, "attempts": [ { "attempt": 1, "error": "The read operation timed out", "elapsed_seconds": 300.053, "completed_at": "2026-09-07T03:09:58.764710+00:00" }, { "attempt": 2, "error": "The read operation timed out", "elapsed_seconds": 300.042, "completed_at": "2026-09-07T03:15:00.817948+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/rent-comps-elm", "expected_target": 3, "expected_decisions": 20 }, { "case_id": "rent-comps-elm", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:06:47.138371+00:00", "completed_at": "2026-09-07T03:07:53.192387+00:00", "elapsed_seconds": 66.054, "cost_usd": 0.19142, "known_partial_cost_usd": 0.19142, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 7014, "completion_tokens": 6254, "total_tokens": 13268, "cost": 0.19142, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1803, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.19142, "gateway_cost": 0.19142 }, "finish_reason": "stop", "elapsed_seconds": 66.042, "completed_at": "2026-09-07T03:07:53.190255+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/rent-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "rent-comps-elm", "task": "rent_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R5361", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-25776", "expected": "ELM-REN-25776", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "annual_face_rent_psf", "actual": 21.0, "expected": 21.0, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 11.28, "expected": 11.279375000000003, "passed": true }, { "field": "term_months", "actual": 36, "expected": 36, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5361" ], "reported_sources": [ "S-R5361" ] }, { "record_id": "R8528", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-95631", "expected": "ELM-REN-95631", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.44, "expected": 22.44, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 16.93, "expected": 16.925612656250006, "passed": true }, { "field": "term_months", "actual": 48, "expected": 48, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8528", "U-R8528-1" ], "reported_sources": [ "S-R8528", "U-R8528-1" ] }, { "record_id": "R4908", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-REN-51407", "expected": "ELM-REN-51407", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "annual_face_rent_psf", "actual": 22.6, "expected": 22.6, "passed": true }, { "field": "effective_net_rent_psf_year", "actual": 19.41, "expected": 19.405271557291673, "passed": true }, { "field": "term_months", "actual": 60, "expected": 60, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4908" ], "reported_sources": [ "S-R4908" ] } ], "decision_checks": [ { "record_id": "R8323", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R5055", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5909", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R6803", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R4908", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7763", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8668", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6572", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3616", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R3778", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1184", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R8329", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5806", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R2825", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9355", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5361", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8578", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R8528", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6603", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R4617", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-fir", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:39:03.808Z", "observed_completed_at": "2026-09-07T02:44:55.287Z", "packet_sha256": "de7c8001706bd0128a46ae12e6f004e6cb0c8c493fdbc21d3b58f8bda5bbd34f", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/rent_comps_dallas_industrial.xlsx", "sha256": "af11653db53252b9875507027a247771b9502df0d50db1f233b84f777aec58fc", "bytes": 8270 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.5429945, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 120.93, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "9b64e8af-173f-42e8-a093-43b0c79d70e2", "session_id": "77224d9b-2122-47b5-9664-aee7744a55b8", "started_at": "2026-09-06T22:40:41.448000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.397313, 0.1456815 ], "total_cost_usd": 0.5429945, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 120.93, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/rent-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "rent-comps-fir", "task": "rent_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2562", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9849", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9791", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7689", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2197", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3079", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4703", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2353", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R1562", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5413", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7010", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R5183", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3428", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6529", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4514", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1617", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6943", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4169", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/rent_comps_dallas_industrial.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Shortlist", "Candidate Decisions", "Summary" ], "table_sheet": "Shortlist", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "af11653db53252b9875507027a247771b9502df0d50db1f233b84f777aec58fc" } }, { "case_id": "rent-comps-fir", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:07:53.195381+00:00", "completed_at": "2026-09-07T03:09:52.805346+00:00", "elapsed_seconds": 119.61, "cost_usd": 0.135687, "known_partial_cost_usd": 0.135687, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6133, "completion_tokens": 5913, "total_tokens": 12046, "cost": 0.06679625, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5632, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06679625, "gateway_cost": 0.06679625 }, "finish_reason": "tool_calls", "elapsed_seconds": 55.157, "completed_at": "2026-09-07T03:08:48.370363+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6415, "completion_tokens": 6764, "total_tokens": 13179, "cost": 0.06889075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6016, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5696, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06889075, "gateway_cost": 0.06889075 }, "finish_reason": "tool_calls", "elapsed_seconds": 63.935, "completed_at": "2026-09-07T03:09:52.803208+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "ae76427beb15ed0b11241c853e3da7226e09e64fcbc0c2a1976a50025b51027f", "bytes": 3861 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/rent-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "rent-comps-fir", "task": "rent_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2562", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9849", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9791", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7689", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2197", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3079", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4703", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2353", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R1562", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5413", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7010", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R5183", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3428", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6529", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4514", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1617", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6943", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4169", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comps" ], "table_sheet": "Comps", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "ae76427beb15ed0b11241c853e3da7226e09e64fcbc0c2a1976a50025b51027f" } }, { "case_id": "rent-comps-fir", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:08:13.737426+00:00", "completed_at": "2026-09-07T03:11:51.484593+00:00", "elapsed_seconds": 217.746, "cost_usd": 0.911095, "known_partial_cost_usd": 0.911095, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9583, "completion_tokens": 11730, "total_tokens": 21313, "cost": 0.341165, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2839, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.341165, "gateway_cost": 0.341165 }, "finish_reason": "tool_calls", "elapsed_seconds": 104.515, "completed_at": "2026-09-07T03:09:58.270658+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 19700, "completion_tokens": 10303, "total_tokens": 30003, "cost": 0.356075, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 519, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.356075, "gateway_cost": 0.356075 }, "finish_reason": "tool_calls", "elapsed_seconds": 83.53, "completed_at": "2026-09-07T03:11:22.371446+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 29571, "completion_tokens": 2640, "total_tokens": 32211, "cost": 0.213855, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 19, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.213855, "gateway_cost": 0.213855 }, "finish_reason": "stop", "elapsed_seconds": 28.576, "completed_at": "2026-09-07T03:11:51.482316+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "f6872ac3acbc692a5ea3d58786260552d66fd18e55945ff58a43c947294c2927", "bytes": 8938 }, { "path": "artifacts/underwriting-v2.xlsx", "sha256": "d3c324f0b0c48d1ae38295461c885a5e8acee87a6aa07922c12945bc1256a4a2", "bytes": 8942 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/rent-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "rent-comps-fir", "task": "rent_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2562", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9849", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9791", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7689", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2197", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3079", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4703", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2353", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R1562", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5413", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7010", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R5183", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3428", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6529", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4514", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1617", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6943", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4169", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v2.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Method" ], "empty_shortlist_representation": "Every displayed candidate is explicitly ineligible", "table_sheet": "Comp Grid", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "d3c324f0b0c48d1ae38295461c885a5e8acee87a6aa07922c12945bc1256a4a2" } }, { "case_id": "rent-comps-fir", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:09:52.806346+00:00", "completed_at": "2026-09-07T03:11:26.486419+00:00", "elapsed_seconds": 93.68, "cost_usd": 0.08421375, "known_partial_cost_usd": 0.08421375, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5387, "completion_tokens": 7748, "total_tokens": 13135, "cost": 0.08421375, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6272, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08421375, "gateway_cost": 0.08421375 }, "finish_reason": "stop", "elapsed_seconds": 93.659, "completed_at": "2026-09-07T03:11:26.485017+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/rent-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "rent-comps-fir", "task": "rent_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2562", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9849", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9791", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7689", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2197", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3079", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4703", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2353", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R1562", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5413", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7010", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R5183", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3428", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6529", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4514", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1617", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6943", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4169", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "rent-comps-fir", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:11:26.487277+00:00", "completed_at": "2026-09-07T03:12:06.498590+00:00", "elapsed_seconds": 40.011, "cost_usd": 0.13219, "known_partial_cost_usd": 0.13219, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 7298, "completion_tokens": 3828, "total_tokens": 11126, "cost": 0.13219, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 822, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.13219, "gateway_cost": 0.13219 }, "finish_reason": "stop", "elapsed_seconds": 39.992, "completed_at": "2026-09-07T03:12:06.496879+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "rent_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/rent-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "rent-comps-fir", "task": "rent_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R1207", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R2562", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9849", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R9791", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown gross expense deduction" ] }, { "record_id": "R7689", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown term_months" ] }, { "record_id": "R2197", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified rent" ] }, { "record_id": "R3079", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4703", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2353", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown ti_allowance_psf" ] }, { "record_id": "R1562", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5413", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7010", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R5183", "actual": false, "expected": false, "passed": true, "reasons": [ "Unknown free_months" ] }, { "record_id": "R3428", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6529", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4514", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1617", "actual": false, "expected": false, "passed": true, "reasons": [ "Lease not executed" ] }, { "record_id": "R6943", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4169", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-alder", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:17:39.204Z", "observed_completed_at": "2026-09-07T02:20:57.986Z", "packet_sha256": "633c47bdaddc9fa78eddc9307cbd8655bacc0ae5c68e7a9efe2bba3ea2b22810", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/alder_charlotte_multifamily_comp_grid.xlsx", "sha256": "356935759940d747e67c864f7e71ce852c4688e73432ef130078f576a9f33f4a", "bytes": 9697 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.64553875, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 121.85, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "02c63899-1855-4d02-b8e1-f42afab28a87", "session_id": "3bf5f49c-ac75-48c0-b733-2d59005813d8", "started_at": "2026-09-06T22:19:11.306000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.47410875, 0.17143 ], "total_cost_usd": 0.64553875, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 121.85, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sales-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-alder", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R8631", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 24000000.0, "expected": 24000000.0, "passed": true }, { "field": "property_size", "actual": 64, "expected": 64, "passed": true }, { "field": "price_per_unit", "actual": 375000.0, "expected": 375000.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8631" ], "reported_sources": [ "S-R8631", "S-R5793" ] }, { "record_id": "R2721", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 27900000.0, "expected": 27900000.0, "passed": true }, { "field": "property_size", "actual": 69, "expected": 69, "passed": true }, { "field": "price_per_unit", "actual": 404347.83, "expected": 404347.82608695654, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2721", "U-R2721-1" ], "reported_sources": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 27600000.0, "expected": 27600000.0, "passed": true }, { "field": "property_size", "actual": 74, "expected": 74, "passed": true }, { "field": "price_per_unit", "actual": 372972.97, "expected": 372972.97297297296, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1038" ], "reported_sources": [ "S-R1038" ] }, { "record_id": "R4976", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 29400000.0, "expected": 29400000.0, "passed": true }, { "field": "property_size", "actual": 78, "expected": 78, "passed": true }, { "field": "price_per_unit", "actual": 376923.08, "expected": 376923.07692307694, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4976" ], "reported_sources": [ "S-R4976" ] }, { "record_id": "R1363", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 31200000.0, "expected": 31200000.0, "passed": true }, { "field": "property_size", "actual": 83, "expected": 83, "passed": true }, { "field": "price_per_unit", "actual": 375903.61, "expected": 375903.6144578313, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1363" ], "reported_sources": [ "S-R1363" ] } ], "decision_checks": [ { "record_id": "R9371", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8904", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R2375", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7563", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R8631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1527", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5793", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1267", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R9650", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2721", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R1038", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3116", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5368", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4976", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1363", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8928", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9770", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5127", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R5520", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/alder_charlotte_multifamily_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions", "Summary & Limitations" ], "table_sheet": "Comp Grid", "original_record_order": [ "R8631", "R2721", "R1038", "R4976", "R1363" ], "answer_record_order": [ "R8631", "R2721", "R1038", "R4976", "R1363" ], "field_checks": [ { "record_id": "R8631", "field": "entity_id", "actual": "ALDER-SAL-26768", "original": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "record_id": "R8631", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R8631", "field": "consideration_usd", "actual": 24000000, "original": 24000000, "expected": 24000000.0, "passed": true }, { "record_id": "R8631", "field": "property_size", "actual": 64, "original": 64, "expected": 64, "passed": true }, { "record_id": "R8631", "field": "price_per_unit", "actual": 375000, "original": 375000, "expected": 375000.0, "passed": true }, { "record_id": "R2721", "field": "entity_id", "actual": "ALDER-SAL-30580", "original": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "record_id": "R2721", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R2721", "field": "consideration_usd", "actual": 27900000, "original": 27900000, "expected": 27900000.0, "passed": true }, { "record_id": "R2721", "field": "property_size", "actual": 69, "original": 69, "expected": 69, "passed": true }, { "record_id": "R2721", "field": "price_per_unit", "actual": 404347.83, "original": 404347.83, "expected": 404347.83, "passed": true }, { "record_id": "R1038", "field": "entity_id", "actual": "ALDER-SAL-50728", "original": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "record_id": "R1038", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1038", "field": "consideration_usd", "actual": 27600000, "original": 27600000, "expected": 27600000.0, "passed": true }, { "record_id": "R1038", "field": "property_size", "actual": 74, "original": 74, "expected": 74, "passed": true }, { "record_id": "R1038", "field": "price_per_unit", "actual": 372972.97, "original": 372972.97, "expected": 372972.97, "passed": true }, { "record_id": "R4976", "field": "entity_id", "actual": "ALDER-SAL-87412", "original": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "record_id": "R4976", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4976", "field": "consideration_usd", "actual": 29400000, "original": 29400000, "expected": 29400000.0, "passed": true }, { "record_id": "R4976", "field": "property_size", "actual": 78, "original": 78, "expected": 78, "passed": true }, { "record_id": "R4976", "field": "price_per_unit", "actual": 376923.08, "original": 376923.08, "expected": 376923.08, "passed": true }, { "record_id": "R1363", "field": "entity_id", "actual": "ALDER-SAL-96959", "original": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "record_id": "R1363", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1363", "field": "consideration_usd", "actual": 31200000, "original": 31200000, "expected": 31200000.0, "passed": true }, { "record_id": "R1363", "field": "property_size", "actual": 83, "original": 83, "expected": 83, "passed": true }, { "record_id": "R1363", "field": "price_per_unit", "actual": 375903.61, "original": 375903.61, "expected": 375903.61, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R8631", "passed": true, "expected": [ "S-R5793", "S-R8631" ], "actual": [ "S-R5793", "S-R8631" ] }, { "record_id": "R2721", "passed": true, "expected": [ "S-R2721", "U-R2721-1" ], "actual": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "passed": true, "expected": [ "S-R1038" ], "actual": [ "S-R1038", "U-R1038-2" ] }, { "record_id": "R4976", "passed": true, "expected": [ "S-R4976" ], "actual": [ "S-R4976", "U-R4976-3" ] }, { "record_id": "R1363", "passed": true, "expected": [ "S-R1363" ], "actual": [ "S-R1363" ] } ], "source_sha256": "356935759940d747e67c864f7e71ce852c4688e73432ef130078f576a9f33f4a" } }, { "case_id": "sales-comps-alder", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:17:10.907881+00:00", "completed_at": "2026-09-07T02:21:07.381157+00:00", "elapsed_seconds": 236.473, "cost_usd": 0.203075, "known_partial_cost_usd": 0.203075, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5047, "completion_tokens": 4529, "total_tokens": 9576, "cost": 0.05159875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4288, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05159875, "gateway_cost": 0.05159875 }, "finish_reason": "tool_calls", "elapsed_seconds": 61.208, "completed_at": "2026-09-07T02:18:12.126511+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5364, "completion_tokens": 6178, "total_tokens": 11542, "cost": 0.062869, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 4992, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5248, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.062869, "gateway_cost": 0.062869 }, "finish_reason": "tool_calls", "elapsed_seconds": 73.371, "completed_at": "2026-09-07T02:19:25.502075+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6305, "completion_tokens": 8663, "total_tokens": 14968, "cost": 0.08860725, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5248, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6720, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08860725, "gateway_cost": 0.08860725 }, "finish_reason": "tool_calls", "elapsed_seconds": 101.143, "completed_at": "2026-09-07T02:21:07.377616+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "d01ba4f1425d1782cca09477a23354a59183bf18635f3058bf5d3963cb265923", "bytes": 4296 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sales-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-alder", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5793", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 24000000, "expected": 24000000.0, "passed": true }, { "field": "property_size", "actual": 64, "expected": 64, "passed": true }, { "field": "price_per_unit", "actual": 375000.0, "expected": 375000.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5793" ], "reported_sources": [ "S-R5793" ] }, { "record_id": "R2721", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 27900000, "expected": 27900000.0, "passed": true }, { "field": "property_size", "actual": 69, "expected": 69, "passed": true }, { "field": "price_per_unit", "actual": 404347.83, "expected": 404347.82608695654, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2721", "U-R2721-1" ], "reported_sources": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 27600000, "expected": 27600000.0, "passed": true }, { "field": "property_size", "actual": 74, "expected": 74, "passed": true }, { "field": "price_per_unit", "actual": 372972.97, "expected": 372972.97297297296, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1038" ], "reported_sources": [ "S-R1038" ] }, { "record_id": "R4976", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 29400000, "expected": 29400000.0, "passed": true }, { "field": "property_size", "actual": 78, "expected": 78, "passed": true }, { "field": "price_per_unit", "actual": 376923.08, "expected": 376923.07692307694, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4976" ], "reported_sources": [ "S-R4976" ] }, { "record_id": "R1363", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 31200000, "expected": 31200000.0, "passed": true }, { "field": "property_size", "actual": 83, "expected": 83, "passed": true }, { "field": "price_per_unit", "actual": 375903.61, "expected": 375903.6144578313, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1363" ], "reported_sources": [ "S-R1363" ] } ], "decision_checks": [ { "record_id": "R9371", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8904", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R2375", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7563", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R8631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1527", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5793", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1267", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R9650", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2721", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R1038", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3116", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5368", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4976", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1363", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8928", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9770", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5127", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R5520", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R5793", "R2721", "R1038", "R4976", "R1363" ], "answer_record_order": [ "R5793", "R2721", "R1038", "R4976", "R1363" ], "field_checks": [ { "record_id": "R5793", "field": "entity_id", "actual": "ALDER-SAL-26768", "original": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "record_id": "R5793", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5793", "field": "consideration_usd", "actual": 24000000, "original": 24000000, "expected": 24000000, "passed": true }, { "record_id": "R5793", "field": "property_size", "actual": 64, "original": 64, "expected": 64, "passed": true }, { "record_id": "R5793", "field": "price_per_unit", "actual": 375000, "original": 375000, "expected": 375000.0, "passed": true }, { "record_id": "R2721", "field": "entity_id", "actual": "ALDER-SAL-30580", "original": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "record_id": "R2721", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R2721", "field": "consideration_usd", "actual": 27900000, "original": 27900000, "expected": 27900000, "passed": true }, { "record_id": "R2721", "field": "property_size", "actual": 69, "original": 69, "expected": 69, "passed": true }, { "record_id": "R2721", "field": "price_per_unit", "actual": 404347.83, "original": 404347.83, "expected": 404347.83, "passed": true }, { "record_id": "R1038", "field": "entity_id", "actual": "ALDER-SAL-50728", "original": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "record_id": "R1038", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1038", "field": "consideration_usd", "actual": 27600000, "original": 27600000, "expected": 27600000, "passed": true }, { "record_id": "R1038", "field": "property_size", "actual": 74, "original": 74, "expected": 74, "passed": true }, { "record_id": "R1038", "field": "price_per_unit", "actual": 372972.97, "original": 372972.97, "expected": 372972.97, "passed": true }, { "record_id": "R4976", "field": "entity_id", "actual": "ALDER-SAL-87412", "original": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "record_id": "R4976", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4976", "field": "consideration_usd", "actual": 29400000, "original": 29400000, "expected": 29400000, "passed": true }, { "record_id": "R4976", "field": "property_size", "actual": 78, "original": 78, "expected": 78, "passed": true }, { "record_id": "R4976", "field": "price_per_unit", "actual": 376923.08, "original": 376923.08, "expected": 376923.08, "passed": true }, { "record_id": "R1363", "field": "entity_id", "actual": "ALDER-SAL-96959", "original": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "record_id": "R1363", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1363", "field": "consideration_usd", "actual": 31200000, "original": 31200000, "expected": 31200000, "passed": true }, { "record_id": "R1363", "field": "property_size", "actual": 83, "original": 83, "expected": 83, "passed": true }, { "record_id": "R1363", "field": "price_per_unit", "actual": 375903.61, "original": 375903.61, "expected": 375903.61, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5793", "passed": true, "expected": [ "S-R5793" ], "actual": [ "S-R5793" ] }, { "record_id": "R2721", "passed": true, "expected": [ "S-R2721", "U-R2721-1" ], "actual": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "passed": true, "expected": [ "S-R1038" ], "actual": [ "S-R1038" ] }, { "record_id": "R4976", "passed": true, "expected": [ "S-R4976" ], "actual": [ "S-R4976" ] }, { "record_id": "R1363", "passed": true, "expected": [ "S-R1363" ], "actual": [ "S-R1363" ] } ], "source_sha256": "d01ba4f1425d1782cca09477a23354a59183bf18635f3058bf5d3963cb265923" } }, { "case_id": "sales-comps-alder", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:17:10.907656+00:00", "completed_at": "2026-09-07T02:18:46.717867+00:00", "elapsed_seconds": 95.81, "cost_usd": 0.377965, "known_partial_cost_usd": 0.377965, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8155, "completion_tokens": 1713, "total_tokens": 9868, "cost": 0.0836, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1091, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0836, "gateway_cost": 0.0836 }, "finish_reason": "tool_calls", "elapsed_seconds": 22.626, "completed_at": "2026-09-07T02:17:33.544903+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 9001, "completion_tokens": 2909, "total_tokens": 11910, "cost": 0.11773, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 168, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11773, "gateway_cost": 0.11773 }, "finish_reason": "tool_calls", "elapsed_seconds": 25.591, "completed_at": "2026-09-07T02:17:59.141739+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 11932, "completion_tokens": 4679, "total_tokens": 16611, "cost": 0.176635, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 579, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.176635, "gateway_cost": 0.176635 }, "finish_reason": "stop", "elapsed_seconds": 46.95, "completed_at": "2026-09-07T02:18:46.716191+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "9968ba27aa5a3c22f79e401b6d8635df6f81b953e745b91cbd48ad59031f1294", "bytes": 593 }, { "path": "artifacts/underwriting-v1.xlsx", "sha256": "0ebf92cab91e195364bdf1dd4e06df09c71b0ee4e51514cf49b3db0f33fca159", "bytes": 4856 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sales-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-alder", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5793", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 24000000, "expected": 24000000.0, "passed": true }, { "field": "property_size", "actual": 64, "expected": 64, "passed": true }, { "field": "price_per_unit", "actual": 375000.0, "expected": 375000.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5793" ], "reported_sources": [ "S-R5793", "S-R8631" ] }, { "record_id": "R2721", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 27900000, "expected": 27900000.0, "passed": true }, { "field": "property_size", "actual": 69, "expected": 69, "passed": true }, { "field": "price_per_unit", "actual": 404347.83, "expected": 404347.82608695654, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2721", "U-R2721-1" ], "reported_sources": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 27600000, "expected": 27600000.0, "passed": true }, { "field": "property_size", "actual": 74, "expected": 74, "passed": true }, { "field": "price_per_unit", "actual": 372972.97, "expected": 372972.97297297296, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1038" ], "reported_sources": [ "S-R1038" ] }, { "record_id": "R4976", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 29400000, "expected": 29400000.0, "passed": true }, { "field": "property_size", "actual": 78, "expected": 78, "passed": true }, { "field": "price_per_unit", "actual": 376923.08, "expected": 376923.07692307694, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4976" ], "reported_sources": [ "S-R4976" ] }, { "record_id": "R1363", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 31200000, "expected": 31200000.0, "passed": true }, { "field": "property_size", "actual": 83, "expected": 83, "passed": true }, { "field": "price_per_unit", "actual": 375903.61, "expected": 375903.6144578313, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1363" ], "reported_sources": [ "S-R1363" ] } ], "decision_checks": [ { "record_id": "R9371", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8904", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R2375", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7563", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R8631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1527", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5793", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1267", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R9650", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2721", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R1038", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3116", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5368", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4976", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1363", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8928", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9770", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5127", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R5520", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R5793", "R2721", "R1038", "R4976", "R1363" ], "answer_record_order": [ "R5793", "R2721", "R1038", "R4976", "R1363" ], "field_checks": [ { "record_id": "R5793", "field": "entity_id", "actual": "ALDER-SAL-26768", "original": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "record_id": "R5793", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5793", "field": "consideration_usd", "actual": 24000000, "original": 24000000, "expected": 24000000, "passed": true }, { "record_id": "R5793", "field": "property_size", "actual": 64, "original": 64, "expected": 64, "passed": true }, { "record_id": "R5793", "field": "price_per_unit", "actual": 375000, "original": 375000, "expected": 375000.0, "passed": true }, { "record_id": "R2721", "field": "entity_id", "actual": "ALDER-SAL-30580", "original": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "record_id": "R2721", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R2721", "field": "consideration_usd", "actual": 27900000, "original": 27900000, "expected": 27900000, "passed": true }, { "record_id": "R2721", "field": "property_size", "actual": 69, "original": 69, "expected": 69, "passed": true }, { "record_id": "R2721", "field": "price_per_unit", "actual": 404347.82608695654, "original": 404347.82608695654, "expected": 404347.83, "passed": true }, { "record_id": "R1038", "field": "entity_id", "actual": "ALDER-SAL-50728", "original": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "record_id": "R1038", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1038", "field": "consideration_usd", "actual": 27600000, "original": 27600000, "expected": 27600000, "passed": true }, { "record_id": "R1038", "field": "property_size", "actual": 74, "original": 74, "expected": 74, "passed": true }, { "record_id": "R1038", "field": "price_per_unit", "actual": 372972.97297297296, "original": 372972.97297297296, "expected": 372972.97, "passed": true }, { "record_id": "R4976", "field": "entity_id", "actual": "ALDER-SAL-87412", "original": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "record_id": "R4976", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4976", "field": "consideration_usd", "actual": 29400000, "original": 29400000, "expected": 29400000, "passed": true }, { "record_id": "R4976", "field": "property_size", "actual": 78, "original": 78, "expected": 78, "passed": true }, { "record_id": "R4976", "field": "price_per_unit", "actual": 376923.07692307694, "original": 376923.07692307694, "expected": 376923.08, "passed": true }, { "record_id": "R1363", "field": "entity_id", "actual": "ALDER-SAL-96959", "original": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "record_id": "R1363", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1363", "field": "consideration_usd", "actual": 31200000, "original": 31200000, "expected": 31200000, "passed": true }, { "record_id": "R1363", "field": "property_size", "actual": 83, "original": 83, "expected": 83, "passed": true }, { "record_id": "R1363", "field": "price_per_unit", "actual": 375903.6144578313, "original": 375903.6144578313, "expected": 375903.61, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5793", "passed": true, "expected": [ "S-R5793", "S-R8631" ], "actual": [ "S-R5793", "S-R8631" ] }, { "record_id": "R2721", "passed": true, "expected": [ "S-R2721", "U-R2721-1" ], "actual": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "passed": true, "expected": [ "S-R1038" ], "actual": [ "S-R1038", "U-R1038-2" ] }, { "record_id": "R4976", "passed": true, "expected": [ "S-R4976" ], "actual": [ "S-R4976", "U-R4976-3" ] }, { "record_id": "R1363", "passed": true, "expected": [ "S-R1363" ], "actual": [ "S-R1363" ] } ], "source_sha256": "0ebf92cab91e195364bdf1dd4e06df09c71b0ee4e51514cf49b3db0f33fca159" } }, { "case_id": "sales-comps-alder", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:17:10.907954+00:00", "completed_at": "2026-09-07T02:19:47.093158+00:00", "elapsed_seconds": 156.185, "cost_usd": 0.09664625, "known_partial_cost_usd": 0.09664625, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4301, "completion_tokens": 9127, "total_tokens": 13428, "cost": 0.09664625, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6976, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.09664625, "gateway_cost": 0.09664625 }, "finish_reason": "stop", "elapsed_seconds": 156.172, "completed_at": "2026-09-07T02:19:47.090833+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sales-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-alder", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5793", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 24000000.0, "expected": 24000000.0, "passed": true }, { "field": "property_size", "actual": 64, "expected": 64, "passed": true }, { "field": "price_per_unit", "actual": 375000.0, "expected": 375000.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5793" ], "reported_sources": [ "S-R5793", "S-R8631" ] }, { "record_id": "R2721", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 27900000.0, "expected": 27900000.0, "passed": true }, { "field": "property_size", "actual": 69, "expected": 69, "passed": true }, { "field": "price_per_unit", "actual": 404348.0, "expected": 404347.82608695654, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2721", "U-R2721-1" ], "reported_sources": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 27600000.0, "expected": 27600000.0, "passed": true }, { "field": "property_size", "actual": 74, "expected": 74, "passed": true }, { "field": "price_per_unit", "actual": 372973.0, "expected": 372972.97297297296, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1038" ], "reported_sources": [ "S-R1038" ] }, { "record_id": "R4976", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 29400000.0, "expected": 29400000.0, "passed": true }, { "field": "property_size", "actual": 78, "expected": 78, "passed": true }, { "field": "price_per_unit", "actual": 376923.0, "expected": 376923.07692307694, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4976" ], "reported_sources": [ "S-R4976" ] }, { "record_id": "R1363", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 31200000.0, "expected": 31200000.0, "passed": true }, { "field": "property_size", "actual": 83, "expected": 83, "passed": true }, { "field": "price_per_unit", "actual": 375904.0, "expected": 375903.6144578313, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1363" ], "reported_sources": [ "S-R1363" ] } ], "decision_checks": [ { "record_id": "R9371", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8904", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R2375", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7563", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R8631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1527", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5793", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1267", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R9650", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2721", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R1038", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3116", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5368", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4976", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1363", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8928", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9770", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5127", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R5520", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-alder", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:18:46.718833+00:00", "completed_at": "2026-09-07T02:19:38.859175+00:00", "elapsed_seconds": 52.14, "cost_usd": 0.158275, "known_partial_cost_usd": 0.158275, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 5870, "completion_tokens": 5157, "total_tokens": 11027, "cost": 0.158275, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 993, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.158275, "gateway_cost": 0.158275 }, "finish_reason": "stop", "elapsed_seconds": 52.118, "completed_at": "2026-09-07T02:19:38.857349+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sales-comps-alder", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-alder", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5793", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-26768", "expected": "ALDER-SAL-26768", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 24000000.0, "expected": 24000000.0, "passed": true }, { "field": "property_size", "actual": 64, "expected": 64, "passed": true }, { "field": "price_per_unit", "actual": 375000.0, "expected": 375000.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5793" ], "reported_sources": [ "S-R5793" ] }, { "record_id": "R2721", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-30580", "expected": "ALDER-SAL-30580", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 27900000.0, "expected": 27900000.0, "passed": true }, { "field": "property_size", "actual": 69, "expected": 69, "passed": true }, { "field": "price_per_unit", "actual": 404347.83, "expected": 404347.82608695654, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2721", "U-R2721-1" ], "reported_sources": [ "S-R2721", "U-R2721-1" ] }, { "record_id": "R1038", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-50728", "expected": "ALDER-SAL-50728", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 27600000.0, "expected": 27600000.0, "passed": true }, { "field": "property_size", "actual": 74, "expected": 74, "passed": true }, { "field": "price_per_unit", "actual": 372972.97, "expected": 372972.97297297296, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1038" ], "reported_sources": [ "S-R1038" ] }, { "record_id": "R4976", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-87412", "expected": "ALDER-SAL-87412", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 29400000.0, "expected": 29400000.0, "passed": true }, { "field": "property_size", "actual": 78, "expected": 78, "passed": true }, { "field": "price_per_unit", "actual": 376923.08, "expected": 376923.07692307694, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4976" ], "reported_sources": [ "S-R4976" ] }, { "record_id": "R1363", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SAL-96959", "expected": "ALDER-SAL-96959", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 31200000.0, "expected": 31200000.0, "passed": true }, { "field": "property_size", "actual": 83, "expected": 83, "passed": true }, { "field": "price_per_unit", "actual": 375903.61, "expected": 375903.6144578313, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1363" ], "reported_sources": [ "S-R1363" ] } ], "decision_checks": [ { "record_id": "R9371", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8904", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R2375", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7563", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R8631", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1527", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5793", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1267", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R9650", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2721", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3130", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R1038", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3116", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5368", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4976", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1363", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8928", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9770", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R5127", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R5520", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-birch", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:19:53.276Z", "observed_completed_at": "2026-09-07T02:26:31.381Z", "packet_sha256": "7a973ac19bfe5ddb6054408b17feaf7b3eba62b5f249260a259d43bcf5af5359", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/birch_phoenix_retail_comp_grid.xlsx", "sha256": "45824c3ee16b864270153ba7a8f2249458a52502af96b53a534590f3f8884c37", "bytes": 8452 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.618047, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 140.54, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "75b86083-c4b9-4a03-981d-77c2aa1fec4b", "session_id": "b76fa4fd-8e60-46ff-960e-1840ebfc8849", "started_at": "2026-09-06T22:21:42.450000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.44060675, 0.17744025 ], "total_cost_usd": 0.618047, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 140.54, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sales-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-birch", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 18, "total": 20 }, "eligible_recall": { "passed": 7, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1958", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 9600000.0, "expected": 9600000.0, "passed": true }, { "field": "property_size", "actual": 24000, "expected": 24000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1958" ], "reported_sources": [ "S-R1958" ] }, { "record_id": "R9866", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 11160000.0, "expected": 11160000.0, "passed": true }, { "field": "property_size", "actual": 25800, "expected": 25800, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9866", "U-R9866-1" ], "reported_sources": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 11040000.0, "expected": 11040000.0, "passed": true }, { "field": "property_size", "actual": 27600, "expected": 27600, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9548" ], "reported_sources": [ "S-R9548" ] }, { "record_id": "R4440", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 11760000.0, "expected": 11760000.0, "passed": true }, { "field": "property_size", "actual": 29400, "expected": 29400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4440" ], "reported_sources": [ "S-R4440" ] }, { "record_id": "R1583", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 12480000.0, "expected": 12480000.0, "passed": true }, { "field": "property_size", "actual": 31200, "expected": 31200, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1583" ], "reported_sources": [ "S-R1583" ] } ], "decision_checks": [ { "record_id": "R4318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4440", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4992", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9548", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9505", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3508", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5005", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R5192", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1583", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1551", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R8670", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2201", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6730", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2037", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R1958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9866", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7511", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R1273", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R5245", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R4762", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/birch_phoenix_retail_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions", "Summary & Limitations" ], "table_sheet": "Comp Grid", "original_record_order": [ "R1958", "R9866", "R9548", "R4440", "R1583" ], "answer_record_order": [ "R1958", "R9866", "R9548", "R4440", "R1583" ], "field_checks": [ { "record_id": "R1958", "field": "entity_id", "actual": "BIRCH-SAL-58018", "original": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "record_id": "R1958", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1958", "field": "consideration_usd", "actual": 9600000, "original": 9600000, "expected": 9600000.0, "passed": true }, { "record_id": "R1958", "field": "property_size", "actual": 24000, "original": 24000, "expected": 24000, "passed": true }, { "record_id": "R1958", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R9866", "field": "entity_id", "actual": "BIRCH-SAL-60411", "original": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "record_id": "R9866", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9866", "field": "consideration_usd", "actual": 11160000, "original": 11160000, "expected": 11160000.0, "passed": true }, { "record_id": "R9866", "field": "property_size", "actual": 25800, "original": 25800, "expected": 25800, "passed": true }, { "record_id": "R9866", "field": "price_per_unit", "actual": 432.56, "original": 432.56, "expected": 432.56, "passed": true }, { "record_id": "R9548", "field": "entity_id", "actual": "BIRCH-SAL-14897", "original": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "record_id": "R9548", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R9548", "field": "consideration_usd", "actual": 11040000, "original": 11040000, "expected": 11040000.0, "passed": true }, { "record_id": "R9548", "field": "property_size", "actual": 27600, "original": 27600, "expected": 27600, "passed": true }, { "record_id": "R9548", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R4440", "field": "entity_id", "actual": "BIRCH-SAL-55425", "original": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "record_id": "R4440", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4440", "field": "consideration_usd", "actual": 11760000, "original": 11760000, "expected": 11760000.0, "passed": true }, { "record_id": "R4440", "field": "property_size", "actual": 29400, "original": 29400, "expected": 29400, "passed": true }, { "record_id": "R4440", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R1583", "field": "entity_id", "actual": "BIRCH-SAL-25516", "original": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "record_id": "R1583", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1583", "field": "consideration_usd", "actual": 12480000, "original": 12480000, "expected": 12480000.0, "passed": true }, { "record_id": "R1583", "field": "property_size", "actual": 31200, "original": 31200, "expected": 31200, "passed": true }, { "record_id": "R1583", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1958", "passed": true, "expected": [ "S-R1958" ], "actual": [ "S-R1958" ] }, { "record_id": "R9866", "passed": true, "expected": [ "S-R9866", "U-R9866-1" ], "actual": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "passed": true, "expected": [ "S-R9548" ], "actual": [ "S-R9548" ] }, { "record_id": "R4440", "passed": true, "expected": [ "S-R4440" ], "actual": [ "S-R4440" ] }, { "record_id": "R1583", "passed": true, "expected": [ "S-R1583" ], "actual": [ "S-R1583" ] } ], "source_sha256": "45824c3ee16b864270153ba7a8f2249458a52502af96b53a534590f3f8884c37" } }, { "case_id": "sales-comps-birch", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:19:38.860157+00:00", "completed_at": "2026-09-07T02:22:33.389556+00:00", "elapsed_seconds": 174.529, "cost_usd": 0.15883775, "known_partial_cost_usd": 0.15883775, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5054, "completion_tokens": 6714, "total_tokens": 11768, "cost": 0.0734575, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5760, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0734575, "gateway_cost": 0.0734575 }, "finish_reason": "tool_calls", "elapsed_seconds": 83.519, "completed_at": "2026-09-07T02:21:02.400309+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6013, "completion_tokens": 8348, "total_tokens": 14361, "cost": 0.08538025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 4992, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6592, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08538025, "gateway_cost": 0.08538025 }, "finish_reason": "tool_calls", "elapsed_seconds": 90.243, "completed_at": "2026-09-07T02:22:33.386915+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "a6eaca32c479ec64b62e6d6e03f6a52320904e8d2e26edd653723eef50191c04", "bytes": 4241 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sales-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-birch", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1958", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 9600000, "expected": 9600000.0, "passed": true }, { "field": "property_size", "actual": 24000, "expected": 24000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1958" ], "reported_sources": [ "S-R1958" ] }, { "record_id": "R9866", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 11160000, "expected": 11160000.0, "passed": true }, { "field": "property_size", "actual": 25800, "expected": 25800, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9866", "U-R9866-1" ], "reported_sources": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 11040000, "expected": 11040000.0, "passed": true }, { "field": "property_size", "actual": 27600, "expected": 27600, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9548" ], "reported_sources": [ "S-R9548" ] }, { "record_id": "R4440", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 11760000, "expected": 11760000.0, "passed": true }, { "field": "property_size", "actual": 29400, "expected": 29400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4440" ], "reported_sources": [ "S-R4440" ] }, { "record_id": "R1583", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 12480000, "expected": 12480000.0, "passed": true }, { "field": "property_size", "actual": 31200, "expected": 31200, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1583" ], "reported_sources": [ "S-R1583" ] } ], "decision_checks": [ { "record_id": "R4318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4440", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4992", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9548", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9505", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3508", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5192", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1583", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1551", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R8670", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2201", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6730", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2037", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R1958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9866", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7511", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R1273", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R5245", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R4762", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comps" ], "table_sheet": "Comps", "original_record_order": [ "R1958", "R9866", "R9548", "R4440", "R1583" ], "answer_record_order": [ "R1958", "R9866", "R9548", "R4440", "R1583" ], "field_checks": [ { "record_id": "R1958", "field": "entity_id", "actual": "BIRCH-SAL-58018", "original": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "record_id": "R1958", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1958", "field": "consideration_usd", "actual": 9600000, "original": 9600000, "expected": 9600000, "passed": true }, { "record_id": "R1958", "field": "property_size", "actual": 24000, "original": 24000, "expected": 24000, "passed": true }, { "record_id": "R1958", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R9866", "field": "entity_id", "actual": "BIRCH-SAL-60411", "original": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "record_id": "R9866", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9866", "field": "consideration_usd", "actual": 11160000, "original": 11160000, "expected": 11160000, "passed": true }, { "record_id": "R9866", "field": "property_size", "actual": 25800, "original": 25800, "expected": 25800, "passed": true }, { "record_id": "R9866", "field": "price_per_unit", "actual": 432.5581395348837, "original": 432.5581395348837, "expected": 432.56, "passed": true }, { "record_id": "R9548", "field": "entity_id", "actual": "BIRCH-SAL-14897", "original": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "record_id": "R9548", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R9548", "field": "consideration_usd", "actual": 11040000, "original": 11040000, "expected": 11040000, "passed": true }, { "record_id": "R9548", "field": "property_size", "actual": 27600, "original": 27600, "expected": 27600, "passed": true }, { "record_id": "R9548", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R4440", "field": "entity_id", "actual": "BIRCH-SAL-55425", "original": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "record_id": "R4440", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4440", "field": "consideration_usd", "actual": 11760000, "original": 11760000, "expected": 11760000, "passed": true }, { "record_id": "R4440", "field": "property_size", "actual": 29400, "original": 29400, "expected": 29400, "passed": true }, { "record_id": "R4440", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R1583", "field": "entity_id", "actual": "BIRCH-SAL-25516", "original": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "record_id": "R1583", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1583", "field": "consideration_usd", "actual": 12480000, "original": 12480000, "expected": 12480000, "passed": true }, { "record_id": "R1583", "field": "property_size", "actual": 31200, "original": 31200, "expected": 31200, "passed": true }, { "record_id": "R1583", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1958", "passed": true, "expected": [ "S-R1958" ], "actual": [ "S-R1958" ] }, { "record_id": "R9866", "passed": true, "expected": [ "S-R9866", "U-R9866-1" ], "actual": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "passed": true, "expected": [ "S-R9548" ], "actual": [ "S-R9548" ] }, { "record_id": "R4440", "passed": true, "expected": [ "S-R4440" ], "actual": [ "S-R4440" ] }, { "record_id": "R1583", "passed": true, "expected": [ "S-R1583" ], "actual": [ "S-R1583" ] } ], "source_sha256": "a6eaca32c479ec64b62e6d6e03f6a52320904e8d2e26edd653723eef50191c04" } }, { "case_id": "sales-comps-birch", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:19:47.094670+00:00", "completed_at": "2026-09-07T02:21:06.201408+00:00", "elapsed_seconds": 79.106, "cost_usd": 0.338265, "known_partial_cost_usd": 0.338265, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8163, "completion_tokens": 1236, "total_tokens": 9399, "cost": 0.071715, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1078, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.071715, "gateway_cost": 0.071715 }, "finish_reason": "tool_calls", "elapsed_seconds": 14.083, "completed_at": "2026-09-07T02:20:01.194875+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8381, "completion_tokens": 3158, "total_tokens": 11539, "cost": 0.120855, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 876, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.120855, "gateway_cost": 0.120855 }, "finish_reason": "tool_calls", "elapsed_seconds": 29.357, "completed_at": "2026-09-07T02:20:30.556105+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10834, "completion_tokens": 3661, "total_tokens": 14495, "cost": 0.145695, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 173, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.145695, "gateway_cost": 0.145695 }, "finish_reason": "stop", "elapsed_seconds": 35.048, "completed_at": "2026-09-07T02:21:06.198123+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "7c70b9cf1cb09c85f70ddb46ef5ea14e8349726863c099f5cebea67ef09b6f84", "bytes": 4660 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sales-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-birch", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1958", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 9600000, "expected": 9600000.0, "passed": true }, { "field": "property_size", "actual": 24000, "expected": 24000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1958" ], "reported_sources": [ "S-R1958" ] }, { "record_id": "R9866", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 11160000, "expected": 11160000.0, "passed": true }, { "field": "property_size", "actual": 25800, "expected": 25800, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9866", "U-R9866-1" ], "reported_sources": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 11040000, "expected": 11040000.0, "passed": true }, { "field": "property_size", "actual": 27600, "expected": 27600, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9548" ], "reported_sources": [ "S-R9548" ] }, { "record_id": "R4440", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 11760000, "expected": 11760000.0, "passed": true }, { "field": "property_size", "actual": 29400, "expected": 29400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4440" ], "reported_sources": [ "S-R4440" ] }, { "record_id": "R1583", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 12480000, "expected": 12480000.0, "passed": true }, { "field": "property_size", "actual": 31200, "expected": 31200, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1583" ], "reported_sources": [ "S-R1583" ] } ], "decision_checks": [ { "record_id": "R4318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4440", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4992", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9548", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9505", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3508", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5192", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1583", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1551", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R8670", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2201", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6730", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2037", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R1958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9866", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7511", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R1273", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R5245", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R4762", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R1958", "R9866", "R9548", "R4440", "R1583" ], "answer_record_order": [ "R1958", "R9866", "R9548", "R4440", "R1583" ], "field_checks": [ { "record_id": "R1958", "field": "entity_id", "actual": "BIRCH-SAL-58018", "original": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "record_id": "R1958", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1958", "field": "consideration_usd", "actual": 9600000, "original": 9600000, "expected": 9600000, "passed": true }, { "record_id": "R1958", "field": "property_size", "actual": 24000, "original": 24000, "expected": 24000, "passed": true }, { "record_id": "R1958", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R9866", "field": "entity_id", "actual": "BIRCH-SAL-60411", "original": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "record_id": "R9866", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9866", "field": "consideration_usd", "actual": 11160000, "original": 11160000, "expected": 11160000, "passed": true }, { "record_id": "R9866", "field": "property_size", "actual": 25800, "original": 25800, "expected": 25800, "passed": true }, { "record_id": "R9866", "field": "price_per_unit", "actual": 432.5581395348837, "original": 432.5581395348837, "expected": 432.56, "passed": true }, { "record_id": "R9548", "field": "entity_id", "actual": "BIRCH-SAL-14897", "original": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "record_id": "R9548", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R9548", "field": "consideration_usd", "actual": 11040000, "original": 11040000, "expected": 11040000, "passed": true }, { "record_id": "R9548", "field": "property_size", "actual": 27600, "original": 27600, "expected": 27600, "passed": true }, { "record_id": "R9548", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R4440", "field": "entity_id", "actual": "BIRCH-SAL-55425", "original": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "record_id": "R4440", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4440", "field": "consideration_usd", "actual": 11760000, "original": 11760000, "expected": 11760000, "passed": true }, { "record_id": "R4440", "field": "property_size", "actual": 29400, "original": 29400, "expected": 29400, "passed": true }, { "record_id": "R4440", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R1583", "field": "entity_id", "actual": "BIRCH-SAL-25516", "original": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "record_id": "R1583", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R1583", "field": "consideration_usd", "actual": 12480000, "original": 12480000, "expected": 12480000, "passed": true }, { "record_id": "R1583", "field": "property_size", "actual": 31200, "original": 31200, "expected": 31200, "passed": true }, { "record_id": "R1583", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1958", "passed": true, "expected": [ "S-R1958" ], "actual": [ "S-R1958" ] }, { "record_id": "R9866", "passed": true, "expected": [ "S-R9866", "U-R9866-1" ], "actual": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "passed": true, "expected": [ "S-R9548" ], "actual": [ "S-R9548", "U-R9548-2" ] }, { "record_id": "R4440", "passed": true, "expected": [ "S-R4440" ], "actual": [ "S-R4440", "U-R4440-3" ] }, { "record_id": "R1583", "passed": true, "expected": [ "S-R1583" ], "actual": [ "S-R1583" ] } ], "source_sha256": "7c70b9cf1cb09c85f70ddb46ef5ea14e8349726863c099f5cebea67ef09b6f84" } }, { "case_id": "sales-comps-birch", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:21:06.202674+00:00", "completed_at": "2026-09-07T02:22:55.619983+00:00", "elapsed_seconds": 109.417, "cost_usd": 0.087025, "known_partial_cost_usd": 0.087025, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4308, "completion_tokens": 8164, "total_tokens": 12472, "cost": 0.087025, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6016, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.087025, "gateway_cost": 0.087025 }, "finish_reason": "stop", "elapsed_seconds": 109.388, "completed_at": "2026-09-07T02:22:55.617816+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sales-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-birch", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1958", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 9600000.0, "expected": 9600000.0, "passed": true }, { "field": "property_size", "actual": 24000, "expected": 24000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1958" ], "reported_sources": [ "S-R1958" ] }, { "record_id": "R9866", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 11160000.0, "expected": 11160000.0, "passed": true }, { "field": "property_size", "actual": 25800, "expected": 25800, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9866", "U-R9866-1" ], "reported_sources": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 11040000.0, "expected": 11040000.0, "passed": true }, { "field": "property_size", "actual": 27600, "expected": 27600, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9548" ], "reported_sources": [ "S-R9548" ] }, { "record_id": "R4440", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 11760000.0, "expected": 11760000.0, "passed": true }, { "field": "property_size", "actual": 29400, "expected": 29400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4440" ], "reported_sources": [ "S-R4440" ] }, { "record_id": "R1583", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 12480000.0, "expected": 12480000.0, "passed": true }, { "field": "property_size", "actual": 31200, "expected": 31200, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1583" ], "reported_sources": [ "S-R1583" ] } ], "decision_checks": [ { "record_id": "R4318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4440", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4992", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9548", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9505", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3508", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5192", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1583", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1551", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R8670", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2201", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6730", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2037", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R1958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9866", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7511", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R1273", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R5245", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R4762", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-birch", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:21:07.382560+00:00", "completed_at": "2026-09-07T02:21:56.672520+00:00", "elapsed_seconds": 49.29, "cost_usd": 0.157915, "known_partial_cost_usd": 0.157915, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 5878, "completion_tokens": 5141, "total_tokens": 11019, "cost": 0.157915, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 954, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.157915, "gateway_cost": 0.157915 }, "finish_reason": "stop", "elapsed_seconds": 49.274, "completed_at": "2026-09-07T02:21:56.670576+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sales-comps-birch", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-birch", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R1958", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-58018", "expected": "BIRCH-SAL-58018", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 9600000.0, "expected": 9600000.0, "passed": true }, { "field": "property_size", "actual": 24000, "expected": 24000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1958" ], "reported_sources": [ "S-R1958" ] }, { "record_id": "R9866", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-60411", "expected": "BIRCH-SAL-60411", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 11160000.0, "expected": 11160000.0, "passed": true }, { "field": "property_size", "actual": 25800, "expected": 25800, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9866", "U-R9866-1" ], "reported_sources": [ "S-R9866", "U-R9866-1" ] }, { "record_id": "R9548", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-14897", "expected": "BIRCH-SAL-14897", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 11040000.0, "expected": 11040000.0, "passed": true }, { "field": "property_size", "actual": 27600, "expected": 27600, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9548" ], "reported_sources": [ "S-R9548" ] }, { "record_id": "R4440", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-55425", "expected": "BIRCH-SAL-55425", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 11760000.0, "expected": 11760000.0, "passed": true }, { "field": "property_size", "actual": 29400, "expected": 29400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4440" ], "reported_sources": [ "S-R4440" ] }, { "record_id": "R1583", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SAL-25516", "expected": "BIRCH-SAL-25516", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 12480000.0, "expected": 12480000.0, "passed": true }, { "field": "property_size", "actual": 31200, "expected": 31200, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1583" ], "reported_sources": [ "S-R1583" ] } ], "decision_checks": [ { "record_id": "R4318", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4440", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4992", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R9548", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9505", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3508", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5192", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1583", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1551", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R8670", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2201", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6730", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2037", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R1958", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9866", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7511", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R1273", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R5245", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R4762", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-cypress", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:26:48.607Z", "observed_completed_at": "2026-09-07T02:29:45.453Z", "packet_sha256": "4f54dc426bd88be45564725195a4ae30863bac82cdd0923fc9ceeefe47cd97f4", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/cypress_comps.xlsx", "sha256": "441f2605a3bc576221a633566ff22e8b9c3f7849431afa1591cb5447d55e61ae", "bytes": 8031 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.634655, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 151.37, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "8e8860fe-f6aa-4c54-a2fd-f8d78d59274c", "session_id": "782c1a2c-f32f-4bc9-af35-cbc0948b1714", "started_at": "2026-09-06T22:28:01.588000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 5, "model_calls": 3, "generation_costs_usd": [ 0.29287025, 0.23848825, 0.1032965 ], "total_cost_usd": 0.634655, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_expertise_instructions", "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 151.37, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sales-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-cypress", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 7, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R6807", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 8000000.0, "expected": 8000000.0, "passed": true }, { "field": "property_size", "actual": 64000, "expected": 64000, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6807" ], "reported_sources": [ "S-R6807" ] }, { "record_id": "R1278", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 9300000.0, "expected": 9300000.0, "passed": true }, { "field": "property_size", "actual": 68800, "expected": 68800, "passed": true }, { "field": "price_per_unit", "actual": 135.17, "expected": 135.17441860465115, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1278", "U-R1278-1" ], "reported_sources": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 9200000.0, "expected": 9200000.0, "passed": true }, { "field": "property_size", "actual": 73600, "expected": 73600, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3282" ], "reported_sources": [ "S-R3282" ] }, { "record_id": "R2008", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 9800000.0, "expected": 9800000.0, "passed": true }, { "field": "property_size", "actual": 78400, "expected": 78400, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2008" ], "reported_sources": [ "S-R2008" ] }, { "record_id": "R6608", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 10400000.0, "expected": 10400000.0, "passed": true }, { "field": "property_size", "actual": 83200, "expected": 83200, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6608" ], "reported_sources": [ "S-R6608" ] } ], "decision_checks": [ { "record_id": "R8675", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R3282", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1278", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2044", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1596", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4227", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5215", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6807", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2134", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R9925", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8973", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R7955", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R6608", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6518", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5660", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R2008", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8212", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R1485", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1604", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6806", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/cypress_comps.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Notes" ], "table_sheet": "Comp Grid", "original_record_order": [ "R6807", "R1278", "R3282", "R2008", "R6608" ], "answer_record_order": [ "R6807", "R1278", "R3282", "R2008", "R6608" ], "field_checks": [ { "record_id": "R6807", "field": "entity_id", "actual": "CYPRESS-SAL-30526", "original": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "record_id": "R6807", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R6807", "field": "consideration_usd", "actual": 8000000, "original": 8000000, "expected": 8000000.0, "passed": true }, { "record_id": "R6807", "field": "property_size", "actual": 64000, "original": 64000, "expected": 64000, "passed": true }, { "record_id": "R6807", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true }, { "record_id": "R1278", "field": "entity_id", "actual": "CYPRESS-SAL-84892", "original": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "record_id": "R1278", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1278", "field": "consideration_usd", "actual": 9300000, "original": 9300000, "expected": 9300000.0, "passed": true }, { "record_id": "R1278", "field": "property_size", "actual": 68800, "original": 68800, "expected": 68800, "passed": true }, { "record_id": "R1278", "field": "price_per_unit", "actual": 135.174418604651, "original": 135.174418604651, "expected": 135.17, "passed": true }, { "record_id": "R3282", "field": "entity_id", "actual": "CYPRESS-SAL-51360", "original": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "record_id": "R3282", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R3282", "field": "consideration_usd", "actual": 9200000, "original": 9200000, "expected": 9200000.0, "passed": true }, { "record_id": "R3282", "field": "property_size", "actual": 73600, "original": 73600, "expected": 73600, "passed": true }, { "record_id": "R3282", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true }, { "record_id": "R2008", "field": "entity_id", "actual": "CYPRESS-SAL-79686", "original": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "record_id": "R2008", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R2008", "field": "consideration_usd", "actual": 9800000, "original": 9800000, "expected": 9800000.0, "passed": true }, { "record_id": "R2008", "field": "property_size", "actual": 78400, "original": 78400, "expected": 78400, "passed": true }, { "record_id": "R2008", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true }, { "record_id": "R6608", "field": "entity_id", "actual": "CYPRESS-SAL-19485", "original": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "record_id": "R6608", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6608", "field": "consideration_usd", "actual": 10400000, "original": 10400000, "expected": 10400000.0, "passed": true }, { "record_id": "R6608", "field": "property_size", "actual": 83200, "original": 83200, "expected": 83200, "passed": true }, { "record_id": "R6608", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R6807", "passed": true, "expected": [ "S-R6807" ], "actual": [ "S-R6807" ] }, { "record_id": "R1278", "passed": true, "expected": [ "S-R1278", "U-R1278-1" ], "actual": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "passed": true, "expected": [ "S-R3282" ], "actual": [ "S-R3282" ] }, { "record_id": "R2008", "passed": true, "expected": [ "S-R2008" ], "actual": [ "S-R2008" ] }, { "record_id": "R6608", "passed": true, "expected": [ "S-R6608" ], "actual": [ "S-R6608" ] } ], "source_sha256": "441f2605a3bc576221a633566ff22e8b9c3f7849431afa1591cb5447d55e61ae" } }, { "case_id": "sales-comps-cypress", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:11:51.486227+00:00", "completed_at": "2026-09-07T03:15:02.279890+00:00", "elapsed_seconds": 190.793, "cost_usd": 0.20613775, "known_partial_cost_usd": 0.20613775, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5073, "completion_tokens": 5190, "total_tokens": 10263, "cost": 0.05824125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5056, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05824125, "gateway_cost": 0.05824125 }, "finish_reason": "tool_calls", "elapsed_seconds": 53.717, "completed_at": "2026-09-07T03:12:45.221239+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5233, "completion_tokens": 6953, "total_tokens": 12186, "cost": 0.07045525, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 4992, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6016, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07045525, "gateway_cost": 0.07045525 }, "finish_reason": "tool_calls", "elapsed_seconds": 68.903, "completed_at": "2026-09-07T03:13:54.127770+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6169, "completion_tokens": 7549, "total_tokens": 13718, "cost": 0.07744125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5120, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5760, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07744125, "gateway_cost": 0.07744125 }, "finish_reason": "tool_calls", "elapsed_seconds": 67.642, "completed_at": "2026-09-07T03:15:02.277269+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "8bf677c3ad7bf8dab8caacd658a942b9cc4660e018f7c738fb8f206ef82a6b49", "bytes": 4287 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sales-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-cypress", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R6807", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 8000000, "expected": 8000000.0, "passed": true }, { "field": "property_size", "actual": 64000, "expected": 64000, "passed": true }, { "field": "price_per_unit", "actual": 125, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6807" ], "reported_sources": [ "S-R6807" ] }, { "record_id": "R1278", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 9300000, "expected": 9300000.0, "passed": true }, { "field": "property_size", "actual": 68800, "expected": 68800, "passed": true }, { "field": "price_per_unit", "actual": 135.17441860465115, "expected": 135.17441860465115, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1278", "U-R1278-1" ], "reported_sources": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 9200000, "expected": 9200000.0, "passed": true }, { "field": "property_size", "actual": 73600, "expected": 73600, "passed": true }, { "field": "price_per_unit", "actual": 125, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3282" ], "reported_sources": [ "S-R3282" ] }, { "record_id": "R2008", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 9800000, "expected": 9800000.0, "passed": true }, { "field": "property_size", "actual": 78400, "expected": 78400, "passed": true }, { "field": "price_per_unit", "actual": 125, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2008" ], "reported_sources": [ "S-R2008" ] }, { "record_id": "R6608", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 10400000, "expected": 10400000.0, "passed": true }, { "field": "property_size", "actual": 83200, "expected": 83200, "passed": true }, { "field": "price_per_unit", "actual": 125, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6608" ], "reported_sources": [ "S-R6608" ] } ], "decision_checks": [ { "record_id": "R8675", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R3282", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1278", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2044", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1596", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4227", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5215", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6807", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2134", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R9925", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8973", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R7955", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R6608", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6518", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5660", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R2008", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8212", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1485", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1604", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6806", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R6807", "R1278", "R3282", "R2008", "R6608" ], "answer_record_order": [ "R6807", "R1278", "R3282", "R2008", "R6608" ], "field_checks": [ { "record_id": "R6807", "field": "entity_id", "actual": "CYPRESS-SAL-30526", "original": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "record_id": "R6807", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R6807", "field": "consideration_usd", "actual": 8000000, "original": 8000000, "expected": 8000000, "passed": true }, { "record_id": "R6807", "field": "property_size", "actual": 64000, "original": 64000, "expected": 64000, "passed": true }, { "record_id": "R6807", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125, "passed": true }, { "record_id": "R1278", "field": "entity_id", "actual": "CYPRESS-SAL-84892", "original": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "record_id": "R1278", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1278", "field": "consideration_usd", "actual": 9300000, "original": 9300000, "expected": 9300000, "passed": true }, { "record_id": "R1278", "field": "property_size", "actual": 68800, "original": 68800, "expected": 68800, "passed": true }, { "record_id": "R1278", "field": "price_per_unit", "actual": 135.17, "original": 135.17, "expected": 135.17441860465115, "passed": true }, { "record_id": "R3282", "field": "entity_id", "actual": "CYPRESS-SAL-51360", "original": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "record_id": "R3282", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R3282", "field": "consideration_usd", "actual": 9200000, "original": 9200000, "expected": 9200000, "passed": true }, { "record_id": "R3282", "field": "property_size", "actual": 73600, "original": 73600, "expected": 73600, "passed": true }, { "record_id": "R3282", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125, "passed": true }, { "record_id": "R2008", "field": "entity_id", "actual": "CYPRESS-SAL-79686", "original": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "record_id": "R2008", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R2008", "field": "consideration_usd", "actual": 9800000, "original": 9800000, "expected": 9800000, "passed": true }, { "record_id": "R2008", "field": "property_size", "actual": 78400, "original": 78400, "expected": 78400, "passed": true }, { "record_id": "R2008", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125, "passed": true }, { "record_id": "R6608", "field": "entity_id", "actual": "CYPRESS-SAL-19485", "original": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "record_id": "R6608", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6608", "field": "consideration_usd", "actual": 10400000, "original": 10400000, "expected": 10400000, "passed": true }, { "record_id": "R6608", "field": "property_size", "actual": 83200, "original": 83200, "expected": 83200, "passed": true }, { "record_id": "R6608", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R6807", "passed": true, "expected": [ "S-R6807" ], "actual": [ "S-R6807" ] }, { "record_id": "R1278", "passed": true, "expected": [ "S-R1278", "U-R1278-1" ], "actual": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "passed": true, "expected": [ "S-R3282" ], "actual": [ "S-R3282" ] }, { "record_id": "R2008", "passed": true, "expected": [ "S-R2008" ], "actual": [ "S-R2008" ] }, { "record_id": "R6608", "passed": true, "expected": [ "S-R6608" ], "actual": [ "S-R6608" ] } ], "source_sha256": "8bf677c3ad7bf8dab8caacd658a942b9cc4660e018f7c738fb8f206ef82a6b49" } }, { "case_id": "sales-comps-cypress", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:12:06.499613+00:00", "completed_at": "2026-09-07T03:13:29.940150+00:00", "elapsed_seconds": 83.44, "cost_usd": 0.355445, "known_partial_cost_usd": 0.355445, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8209, "completion_tokens": 1037, "total_tokens": 9246, "cost": 0.06697, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 879, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06697, "gateway_cost": 0.06697 }, "finish_reason": "tool_calls", "elapsed_seconds": 10.635, "completed_at": "2026-09-07T03:12:17.154082+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8427, "completion_tokens": 3192, "total_tokens": 11619, "cost": 0.121935, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 882, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.121935, "gateway_cost": 0.121935 }, "finish_reason": "tool_calls", "elapsed_seconds": 28.336, "completed_at": "2026-09-07T03:12:45.496689+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10893, "completion_tokens": 4483, "total_tokens": 15376, "cost": 0.16654, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 403, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.16654, "gateway_cost": 0.16654 }, "finish_reason": "stop", "elapsed_seconds": 43.922, "completed_at": "2026-09-07T03:13:29.937630+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "c7f9fa53f918377ef0c79589b1f0d8bc3480a4b21404305d2810446c9cf16835", "bytes": 4657 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sales-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-cypress", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R6807", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 8000000, "expected": 8000000.0, "passed": true }, { "field": "property_size", "actual": 64000, "expected": 64000, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6807" ], "reported_sources": [ "S-R6807", "S-R8212" ] }, { "record_id": "R1278", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 9300000, "expected": 9300000.0, "passed": true }, { "field": "property_size", "actual": 68800, "expected": 68800, "passed": true }, { "field": "price_per_unit", "actual": 135.17, "expected": 135.17441860465115, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1278", "U-R1278-1" ], "reported_sources": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 9200000, "expected": 9200000.0, "passed": true }, { "field": "property_size", "actual": 73600, "expected": 73600, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3282" ], "reported_sources": [ "S-R3282" ] }, { "record_id": "R2008", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 9800000, "expected": 9800000.0, "passed": true }, { "field": "property_size", "actual": 78400, "expected": 78400, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2008" ], "reported_sources": [ "S-R2008" ] }, { "record_id": "R6608", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 10400000, "expected": 10400000.0, "passed": true }, { "field": "property_size", "actual": 83200, "expected": 83200, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6608" ], "reported_sources": [ "S-R6608" ] } ], "decision_checks": [ { "record_id": "R8675", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R3282", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1278", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2044", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1596", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4227", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5215", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6807", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2134", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R9925", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8973", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R7955", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R6608", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6518", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5660", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R2008", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8212", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1485", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1604", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6806", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [ { "record_id": "R6807", "field": "property_size", "original": { "value": 64000, "unit": "sf" }, "normalized": 64000, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R6807", "field": "price_per_unit", "original": { "value": 125.0, "unit": "USD_per_sf" }, "normalized": 125.0, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R1278", "field": "property_size", "original": { "value": 68800, "unit": "sf" }, "normalized": 68800, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R1278", "field": "price_per_unit", "original": { "value": 135.17, "unit": "USD_per_sf" }, "normalized": 135.17, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R3282", "field": "property_size", "original": { "value": 73600, "unit": "sf" }, "normalized": 73600, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R3282", "field": "price_per_unit", "original": { "value": 125.0, "unit": "USD_per_sf" }, "normalized": 125.0, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R2008", "field": "property_size", "original": { "value": 78400, "unit": "sf" }, "normalized": 78400, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R2008", "field": "price_per_unit", "original": { "value": 125.0, "unit": "USD_per_sf" }, "normalized": 125.0, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R6608", "field": "property_size", "original": { "value": 83200, "unit": "sf" }, "normalized": 83200, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R6608", "field": "price_per_unit", "original": { "value": 125.0, "unit": "USD_per_sf" }, "normalized": 125.0, "reason": "Explicit numeric wrapper with the unit required by the brief" } ] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R6807", "R1278", "R3282", "R2008", "R6608" ], "answer_record_order": [ "R6807", "R1278", "R3282", "R2008", "R6608" ], "field_checks": [ { "record_id": "R6807", "field": "entity_id", "actual": "CYPRESS-SAL-30526", "original": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "record_id": "R6807", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R6807", "field": "consideration_usd", "actual": 8000000, "original": 8000000, "expected": 8000000, "passed": true }, { "record_id": "R6807", "field": "property_size", "actual": 64000, "original": 64000, "expected": 64000, "passed": true }, { "record_id": "R6807", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true }, { "record_id": "R1278", "field": "entity_id", "actual": "CYPRESS-SAL-84892", "original": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "record_id": "R1278", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1278", "field": "consideration_usd", "actual": 9300000, "original": 9300000, "expected": 9300000, "passed": true }, { "record_id": "R1278", "field": "property_size", "actual": 68800, "original": 68800, "expected": 68800, "passed": true }, { "record_id": "R1278", "field": "price_per_unit", "actual": 135.17441860465115, "original": 135.17441860465115, "expected": 135.17, "passed": true }, { "record_id": "R3282", "field": "entity_id", "actual": "CYPRESS-SAL-51360", "original": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "record_id": "R3282", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R3282", "field": "consideration_usd", "actual": 9200000, "original": 9200000, "expected": 9200000, "passed": true }, { "record_id": "R3282", "field": "property_size", "actual": 73600, "original": 73600, "expected": 73600, "passed": true }, { "record_id": "R3282", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true }, { "record_id": "R2008", "field": "entity_id", "actual": "CYPRESS-SAL-79686", "original": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "record_id": "R2008", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R2008", "field": "consideration_usd", "actual": 9800000, "original": 9800000, "expected": 9800000, "passed": true }, { "record_id": "R2008", "field": "property_size", "actual": 78400, "original": 78400, "expected": 78400, "passed": true }, { "record_id": "R2008", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true }, { "record_id": "R6608", "field": "entity_id", "actual": "CYPRESS-SAL-19485", "original": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "record_id": "R6608", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6608", "field": "consideration_usd", "actual": 10400000, "original": 10400000, "expected": 10400000, "passed": true }, { "record_id": "R6608", "field": "property_size", "actual": 83200, "original": 83200, "expected": 83200, "passed": true }, { "record_id": "R6608", "field": "price_per_unit", "actual": 125, "original": 125, "expected": 125.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R6807", "passed": true, "expected": [ "S-R6807", "S-R8212" ], "actual": [ "S-R6807", "S-R8212" ] }, { "record_id": "R1278", "passed": true, "expected": [ "S-R1278", "U-R1278-1" ], "actual": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "passed": true, "expected": [ "S-R3282" ], "actual": [ "S-R3282", "U-R3282-2" ] }, { "record_id": "R2008", "passed": true, "expected": [ "S-R2008" ], "actual": [ "S-R2008", "U-R2008-3" ] }, { "record_id": "R6608", "passed": true, "expected": [ "S-R6608" ], "actual": [ "S-R6608" ] } ], "source_sha256": "c7f9fa53f918377ef0c79589b1f0d8bc3480a4b21404305d2810446c9cf16835" } }, { "case_id": "sales-comps-cypress", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:13:29.941244+00:00", "completed_at": "2026-09-07T03:16:30.228063+00:00", "elapsed_seconds": 180.286, "cost_usd": 0.10219875, "known_partial_cost_usd": 0.10219875, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4327, "completion_tokens": 9679, "total_tokens": 14006, "cost": 0.10219875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7232, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10219875, "gateway_cost": 0.10219875 }, "finish_reason": "stop", "elapsed_seconds": 180.268, "completed_at": "2026-09-07T03:16:30.226701+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sales-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-cypress", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R6807", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 8000000.0, "expected": 8000000.0, "passed": true }, { "field": "property_size", "actual": 64000, "expected": 64000, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6807" ], "reported_sources": [ "S-R6807" ] }, { "record_id": "R1278", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 9300000.0, "expected": 9300000.0, "passed": true }, { "field": "property_size", "actual": 68800, "expected": 68800, "passed": true }, { "field": "price_per_unit", "actual": 135.17, "expected": 135.17441860465115, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1278", "U-R1278-1" ], "reported_sources": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 9200000.0, "expected": 9200000.0, "passed": true }, { "field": "property_size", "actual": 73600, "expected": 73600, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3282" ], "reported_sources": [ "S-R3282" ] }, { "record_id": "R2008", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 9800000.0, "expected": 9800000.0, "passed": true }, { "field": "property_size", "actual": 78400, "expected": 78400, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2008" ], "reported_sources": [ "S-R2008" ] }, { "record_id": "R6608", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 10400000.0, "expected": 10400000.0, "passed": true }, { "field": "property_size", "actual": 83200, "expected": 83200, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6608" ], "reported_sources": [ "S-R6608" ] } ], "decision_checks": [ { "record_id": "R8675", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R3282", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1278", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2044", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1596", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4227", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5215", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6807", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2134", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R9925", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8973", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R7955", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R6608", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6518", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5660", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R2008", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8212", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1485", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1604", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6806", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-cypress", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:15:00.820364+00:00", "completed_at": "2026-09-07T03:15:59.171716+00:00", "elapsed_seconds": 58.351, "cost_usd": 0.17522, "known_partial_cost_usd": 0.17522, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 5924, "completion_tokens": 5824, "total_tokens": 11748, "cost": 0.17522, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1138, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.17522, "gateway_cost": 0.17522 }, "finish_reason": "stop", "elapsed_seconds": 58.333, "completed_at": "2026-09-07T03:15:59.169772+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sales-comps-cypress", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-cypress", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R6807", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-30526", "expected": "CYPRESS-SAL-30526", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 8000000.0, "expected": 8000000.0, "passed": true }, { "field": "property_size", "actual": 64000, "expected": 64000, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6807" ], "reported_sources": [ "S-R6807", "S-R8212" ] }, { "record_id": "R1278", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-84892", "expected": "CYPRESS-SAL-84892", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 9300000.0, "expected": 9300000.0, "passed": true }, { "field": "property_size", "actual": 68800, "expected": 68800, "passed": true }, { "field": "price_per_unit", "actual": 135.17, "expected": 135.17441860465115, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1278", "U-R1278-1" ], "reported_sources": [ "S-R1278", "U-R1278-1" ] }, { "record_id": "R3282", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-51360", "expected": "CYPRESS-SAL-51360", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 9200000.0, "expected": 9200000.0, "passed": true }, { "field": "property_size", "actual": 73600, "expected": 73600, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3282" ], "reported_sources": [ "S-R3282" ] }, { "record_id": "R2008", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-79686", "expected": "CYPRESS-SAL-79686", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 9800000.0, "expected": 9800000.0, "passed": true }, { "field": "property_size", "actual": 78400, "expected": 78400, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2008" ], "reported_sources": [ "S-R2008" ] }, { "record_id": "R6608", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SAL-19485", "expected": "CYPRESS-SAL-19485", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 10400000.0, "expected": 10400000.0, "passed": true }, { "field": "property_size", "actual": 83200, "expected": 83200, "passed": true }, { "field": "price_per_unit", "actual": 125.0, "expected": 125.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6608" ], "reported_sources": [ "S-R6608" ] } ], "decision_checks": [ { "record_id": "R8675", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R3282", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1278", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2044", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1596", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4227", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5215", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6807", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2134", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R9925", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8973", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R7955", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R6608", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6518", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R5660", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R2008", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8212", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1485", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R1604", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6806", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-dogwood", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:30:40.205Z", "observed_completed_at": "2026-09-07T02:36:24.997Z", "packet_sha256": "4d50c547109144341d98091017604cc3810727086d6de6369da209dc8b8360e1", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/dogwood_comp_grid.xlsx", "sha256": "d9580028e37fdb68a901969767e726d4a831a7210ba45b3efa237a858e1d76f4", "bytes": 8900 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.72973975, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 162.42, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "4b172979-1cac-48f8-8846-4b63bfcdd61f", "session_id": "356572ec-2366-40b7-8c26-d7178e64af32", "started_at": "2026-09-06T22:32:49.192000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.5450455, 0.18469425 ], "total_cost_usd": 0.72973975, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 162.42, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sales-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-dogwood", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5709", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 28000000.0, "expected": 28000000.0, "passed": true }, { "field": "property_size", "actual": 96, "expected": 96, "passed": true }, { "field": "price_per_unit", "actual": 291666.67, "expected": 291666.6666666667, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5709" ], "reported_sources": [ "S-R5709", "S-R4477" ] }, { "record_id": "R1935", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 32550000.0, "expected": 32550000.0, "passed": true }, { "field": "property_size", "actual": 103, "expected": 103, "passed": true }, { "field": "price_per_unit", "actual": 316019.42, "expected": 316019.41747572814, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1935", "U-R1935-1" ], "reported_sources": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 32200000.0, "expected": 32200000.000000004, "passed": true }, { "field": "property_size", "actual": 110, "expected": 110, "passed": true }, { "field": "price_per_unit", "actual": 292727.27, "expected": 292727.27272727276, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8685" ], "reported_sources": [ "S-R8685" ] }, { "record_id": "R1029", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 34300000.0, "expected": 34300000.0, "passed": true }, { "field": "property_size", "actual": 118, "expected": 118, "passed": true }, { "field": "price_per_unit", "actual": 290677.97, "expected": 290677.9661016949, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1029" ], "reported_sources": [ "S-R1029" ] }, { "record_id": "R4639", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 36400000.0, "expected": 36400000.0, "passed": true }, { "field": "property_size", "actual": 125, "expected": 125, "passed": true }, { "field": "price_per_unit", "actual": 291200.0, "expected": 291200.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4639" ], "reported_sources": [ "S-R4639" ] } ], "decision_checks": [ { "record_id": "R4639", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3713", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4402", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R5751", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1935", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9886", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3929", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R4855", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5709", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1347", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1328", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7446", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8685", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4477", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4014", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3120", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R1029", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4010", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7792", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R1717", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/dogwood_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions", "Summary" ], "table_sheet": "Comp Grid", "original_record_order": [ "R5709", "R1935", "R8685", "R1029", "R4639" ], "answer_record_order": [ "R5709", "R1935", "R8685", "R1029", "R4639" ], "field_checks": [ { "record_id": "R5709", "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "original": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "record_id": "R5709", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5709", "field": "consideration_usd", "actual": 28000000, "original": 28000000, "expected": 28000000.0, "passed": true }, { "record_id": "R5709", "field": "property_size", "actual": 96, "original": 96, "expected": 96, "passed": true }, { "record_id": "R5709", "field": "price_per_unit", "actual": 291666.67, "original": 291666.67, "expected": 291666.67, "passed": true }, { "record_id": "R1935", "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "original": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "record_id": "R1935", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1935", "field": "consideration_usd", "actual": 32550000, "original": 32550000, "expected": 32550000.0, "passed": true }, { "record_id": "R1935", "field": "property_size", "actual": 103, "original": 103, "expected": 103, "passed": true }, { "record_id": "R1935", "field": "price_per_unit", "actual": 316019.42, "original": 316019.42, "expected": 316019.42, "passed": true }, { "record_id": "R8685", "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "original": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "record_id": "R8685", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R8685", "field": "consideration_usd", "actual": 32200000, "original": 32200000, "expected": 32200000.0, "passed": true }, { "record_id": "R8685", "field": "property_size", "actual": 110, "original": 110, "expected": 110, "passed": true }, { "record_id": "R8685", "field": "price_per_unit", "actual": 292727.27, "original": 292727.27, "expected": 292727.27, "passed": true }, { "record_id": "R1029", "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "original": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "record_id": "R1029", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R1029", "field": "consideration_usd", "actual": 34300000, "original": 34300000, "expected": 34300000.0, "passed": true }, { "record_id": "R1029", "field": "property_size", "actual": 118, "original": 118, "expected": 118, "passed": true }, { "record_id": "R1029", "field": "price_per_unit", "actual": 290677.97, "original": 290677.97, "expected": 290677.97, "passed": true }, { "record_id": "R4639", "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "original": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "record_id": "R4639", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R4639", "field": "consideration_usd", "actual": 36400000, "original": 36400000, "expected": 36400000.0, "passed": true }, { "record_id": "R4639", "field": "property_size", "actual": 125, "original": 125, "expected": 125, "passed": true }, { "record_id": "R4639", "field": "price_per_unit", "actual": 291200, "original": 291200, "expected": 291200.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5709", "passed": true, "expected": [ "S-R4477", "S-R5709" ], "actual": [ "S-R4477", "S-R5709" ] }, { "record_id": "R1935", "passed": true, "expected": [ "S-R1935", "U-R1935-1" ], "actual": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "passed": true, "expected": [ "S-R8685" ], "actual": [ "S-R8685" ] }, { "record_id": "R1029", "passed": true, "expected": [ "S-R1029" ], "actual": [ "S-R1029" ] }, { "record_id": "R4639", "passed": true, "expected": [ "S-R4639" ], "actual": [ "S-R4639" ] } ], "source_sha256": "d9580028e37fdb68a901969767e726d4a831a7210ba45b3efa237a858e1d76f4" } }, { "case_id": "sales-comps-dogwood", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:15:02.281217+00:00", "completed_at": "2026-09-07T03:18:38.439471+00:00", "elapsed_seconds": 216.158, "cost_usd": 0.2220065, "known_partial_cost_usd": 0.2220065, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5068, "completion_tokens": 4681, "total_tokens": 9749, "cost": 0.053145, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4544, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.053145, "gateway_cost": 0.053145 }, "finish_reason": "tool_calls", "elapsed_seconds": 43.548, "completed_at": "2026-09-07T03:15:45.848176+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5242, "completion_tokens": 7069, "total_tokens": 12311, "cost": 0.0716265, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 4992, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6080, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0716265, "gateway_cost": 0.0716265 }, "finish_reason": "tool_calls", "elapsed_seconds": 61.655, "completed_at": "2026-09-07T03:16:47.505240+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6244, "completion_tokens": 9519, "total_tokens": 15763, "cost": 0.097235, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5120, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7808, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.097235, "gateway_cost": 0.097235 }, "finish_reason": "tool_calls", "elapsed_seconds": 109.126, "completed_at": "2026-09-07T03:18:38.436851+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "fac353b7218abfdc694b5a30f362e6c7e08947680e3231005dfb34d58c27dfaf", "bytes": 4324 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sales-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-dogwood", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5709", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 28000000, "expected": 28000000.0, "passed": true }, { "field": "property_size", "actual": 96, "expected": 96, "passed": true }, { "field": "price_per_unit", "actual": 291666.67, "expected": 291666.6666666667, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5709" ], "reported_sources": [ "S-R5709" ] }, { "record_id": "R1935", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 32550000, "expected": 32550000.0, "passed": true }, { "field": "property_size", "actual": 103, "expected": 103, "passed": true }, { "field": "price_per_unit", "actual": 316019.42, "expected": 316019.41747572814, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1935", "U-R1935-1" ], "reported_sources": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 32200000, "expected": 32200000.000000004, "passed": true }, { "field": "property_size", "actual": 110, "expected": 110, "passed": true }, { "field": "price_per_unit", "actual": 292727.27, "expected": 292727.27272727276, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8685" ], "reported_sources": [ "S-R8685" ] }, { "record_id": "R1029", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 34300000, "expected": 34300000.0, "passed": true }, { "field": "property_size", "actual": 118, "expected": 118, "passed": true }, { "field": "price_per_unit", "actual": 290677.97, "expected": 290677.9661016949, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1029" ], "reported_sources": [ "S-R1029" ] }, { "record_id": "R4639", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 36400000, "expected": 36400000.0, "passed": true }, { "field": "property_size", "actual": 125, "expected": 125, "passed": true }, { "field": "price_per_unit", "actual": 291200.0, "expected": 291200.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4639" ], "reported_sources": [ "S-R4639" ] } ], "decision_checks": [ { "record_id": "R4639", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3713", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4402", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R5751", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1935", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9886", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3929", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R4855", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5709", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1347", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1328", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7446", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8685", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4477", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4014", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3120", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R1029", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4010", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7792", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R1717", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comps" ], "table_sheet": "Comps", "original_record_order": [ "R5709", "R1935", "R8685", "R1029", "R4639" ], "answer_record_order": [ "R5709", "R1935", "R8685", "R1029", "R4639" ], "field_checks": [ { "record_id": "R5709", "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "original": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "record_id": "R5709", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5709", "field": "consideration_usd", "actual": 28000000, "original": 28000000, "expected": 28000000, "passed": true }, { "record_id": "R5709", "field": "property_size", "actual": 96, "original": 96, "expected": 96, "passed": true }, { "record_id": "R5709", "field": "price_per_unit", "actual": 291666.6666666667, "original": 291666.6666666667, "expected": 291666.67, "passed": true }, { "record_id": "R1935", "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "original": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "record_id": "R1935", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1935", "field": "consideration_usd", "actual": 32550000, "original": 32550000, "expected": 32550000, "passed": true }, { "record_id": "R1935", "field": "property_size", "actual": 103, "original": 103, "expected": 103, "passed": true }, { "record_id": "R1935", "field": "price_per_unit", "actual": 316019.41747572814, "original": 316019.41747572814, "expected": 316019.42, "passed": true }, { "record_id": "R8685", "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "original": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "record_id": "R8685", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R8685", "field": "consideration_usd", "actual": 32200000, "original": 32200000, "expected": 32200000, "passed": true }, { "record_id": "R8685", "field": "property_size", "actual": 110, "original": 110, "expected": 110, "passed": true }, { "record_id": "R8685", "field": "price_per_unit", "actual": 292727.2727272727, "original": 292727.2727272727, "expected": 292727.27, "passed": true }, { "record_id": "R1029", "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "original": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "record_id": "R1029", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R1029", "field": "consideration_usd", "actual": 34300000, "original": 34300000, "expected": 34300000, "passed": true }, { "record_id": "R1029", "field": "property_size", "actual": 118, "original": 118, "expected": 118, "passed": true }, { "record_id": "R1029", "field": "price_per_unit", "actual": 290677.9661016949, "original": 290677.9661016949, "expected": 290677.97, "passed": true }, { "record_id": "R4639", "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "original": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "record_id": "R4639", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R4639", "field": "consideration_usd", "actual": 36400000, "original": 36400000, "expected": 36400000, "passed": true }, { "record_id": "R4639", "field": "property_size", "actual": 125, "original": 125, "expected": 125, "passed": true }, { "record_id": "R4639", "field": "price_per_unit", "actual": 291200, "original": 291200, "expected": 291200.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5709", "passed": true, "expected": [ "S-R5709" ], "actual": [ "S-R5709" ] }, { "record_id": "R1935", "passed": true, "expected": [ "S-R1935", "U-R1935-1" ], "actual": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "passed": true, "expected": [ "S-R8685" ], "actual": [ "S-R8685" ] }, { "record_id": "R1029", "passed": true, "expected": [ "S-R1029" ], "actual": [ "S-R1029" ] }, { "record_id": "R4639", "passed": true, "expected": [ "S-R4639" ], "actual": [ "S-R4639" ] } ], "source_sha256": "fac353b7218abfdc694b5a30f362e6c7e08947680e3231005dfb34d58c27dfaf" } }, { "case_id": "sales-comps-dogwood", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:15:59.173057+00:00", "completed_at": "2026-09-07T03:17:41.146435+00:00", "elapsed_seconds": 101.973, "cost_usd": 0.42435500000000004, "known_partial_cost_usd": 0.42435500000000004, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8246, "completion_tokens": 982, "total_tokens": 9228, "cost": 0.06578, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 809, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06578, "gateway_cost": 0.06578 }, "finish_reason": "tool_calls", "elapsed_seconds": 11.654, "completed_at": "2026-09-07T03:16:10.862414+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8518, "completion_tokens": 6692, "total_tokens": 15210, "cost": 0.20989, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1199, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.20989, "gateway_cost": 0.20989 }, "finish_reason": "tool_calls", "elapsed_seconds": 58.378, "completed_at": "2026-09-07T03:17:09.243778+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 14277, "completion_tokens": 3092, "total_tokens": 17369, "cost": 0.148685, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 92, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.148685, "gateway_cost": 0.148685 }, "finish_reason": "stop", "elapsed_seconds": 31.366, "completed_at": "2026-09-07T03:17:41.144387+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "11a1c3675d05312ebd4561bfa5a187c7834c98e91c70f1dbceb385bbbc982961", "bytes": 7070 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sales-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-dogwood", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5709", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 28000000, "expected": 28000000.0, "passed": true }, { "field": "property_size", "actual": 96, "expected": 96, "passed": true }, { "field": "price_per_unit", "actual": 291666.67, "expected": 291666.6666666667, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5709" ], "reported_sources": [ "S-R5709", "S-R4477" ] }, { "record_id": "R1935", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 32550000, "expected": 32550000.0, "passed": true }, { "field": "property_size", "actual": 103, "expected": 103, "passed": true }, { "field": "price_per_unit", "actual": 316019.42, "expected": 316019.41747572814, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1935", "U-R1935-1" ], "reported_sources": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 32200000, "expected": 32200000.000000004, "passed": true }, { "field": "property_size", "actual": 110, "expected": 110, "passed": true }, { "field": "price_per_unit", "actual": 292727.27, "expected": 292727.27272727276, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8685" ], "reported_sources": [ "S-R8685" ] }, { "record_id": "R1029", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 34300000, "expected": 34300000.0, "passed": true }, { "field": "property_size", "actual": 118, "expected": 118, "passed": true }, { "field": "price_per_unit", "actual": 290677.97, "expected": 290677.9661016949, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1029" ], "reported_sources": [ "S-R1029" ] }, { "record_id": "R4639", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 36400000, "expected": 36400000.0, "passed": true }, { "field": "property_size", "actual": 125, "expected": 125, "passed": true }, { "field": "price_per_unit", "actual": 291200.0, "expected": 291200.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4639" ], "reported_sources": [ "S-R4639" ] } ], "decision_checks": [ { "record_id": "R4639", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3713", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4402", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R5751", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1935", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9886", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3929", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R4855", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5709", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1347", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1328", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7446", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8685", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4477", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4014", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3120", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R1029", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4010", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7792", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R1717", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [ { "record_id": "R5709", "field": "property_size", "original": { "value": 96, "unit": "unit" }, "normalized": 96, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R1935", "field": "property_size", "original": { "value": 103, "unit": "unit" }, "normalized": 103, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R8685", "field": "property_size", "original": { "value": 110, "unit": "unit" }, "normalized": 110, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R1029", "field": "property_size", "original": { "value": 118, "unit": "unit" }, "normalized": 118, "reason": "Explicit numeric wrapper with the unit required by the brief" }, { "record_id": "R4639", "field": "property_size", "original": { "value": 125, "unit": "unit" }, "normalized": 125, "reason": "Explicit numeric wrapper with the unit required by the brief" } ] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Assumptions", "Excluded" ], "table_sheet": "Comp Grid", "original_record_order": [ "R5709", "R1935", "R8685", "R1029", "R4639" ], "answer_record_order": [ "R5709", "R1935", "R8685", "R1029", "R4639" ], "field_checks": [ { "record_id": "R5709", "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "original": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "record_id": "R5709", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5709", "field": "consideration_usd", "actual": 28000000, "original": 28000000, "expected": 28000000, "passed": true }, { "record_id": "R5709", "field": "property_size", "actual": 96, "original": 96, "expected": 96, "passed": true }, { "record_id": "R5709", "field": "price_per_unit", "actual": 291666.6666666667, "original": 291666.6666666667, "expected": 291666.67, "passed": true }, { "record_id": "R1935", "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "original": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "record_id": "R1935", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1935", "field": "consideration_usd", "actual": 32550000, "original": 32550000, "expected": 32550000, "passed": true }, { "record_id": "R1935", "field": "property_size", "actual": 103, "original": 103, "expected": 103, "passed": true }, { "record_id": "R1935", "field": "price_per_unit", "actual": 316019.41747572814, "original": 316019.41747572814, "expected": 316019.42, "passed": true }, { "record_id": "R8685", "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "original": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "record_id": "R8685", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R8685", "field": "consideration_usd", "actual": 32200000, "original": 32200000, "expected": 32200000, "passed": true }, { "record_id": "R8685", "field": "property_size", "actual": 110, "original": 110, "expected": 110, "passed": true }, { "record_id": "R8685", "field": "price_per_unit", "actual": 292727.2727272727, "original": 292727.2727272727, "expected": 292727.27, "passed": true }, { "record_id": "R1029", "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "original": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "record_id": "R1029", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R1029", "field": "consideration_usd", "actual": 34300000, "original": 34300000, "expected": 34300000, "passed": true }, { "record_id": "R1029", "field": "property_size", "actual": 118, "original": 118, "expected": 118, "passed": true }, { "record_id": "R1029", "field": "price_per_unit", "actual": 290677.9661016949, "original": 290677.9661016949, "expected": 290677.97, "passed": true }, { "record_id": "R4639", "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "original": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "record_id": "R4639", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R4639", "field": "consideration_usd", "actual": 36400000, "original": 36400000, "expected": 36400000, "passed": true }, { "record_id": "R4639", "field": "property_size", "actual": 125, "original": 125, "expected": 125, "passed": true }, { "record_id": "R4639", "field": "price_per_unit", "actual": 291200, "original": 291200, "expected": 291200.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R5709", "passed": true, "expected": [ "S-R4477", "S-R5709" ], "actual": [ "S-R4477", "S-R5709" ] }, { "record_id": "R1935", "passed": true, "expected": [ "S-R1935", "U-R1935-1" ], "actual": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "passed": true, "expected": [ "S-R8685" ], "actual": [ "S-R8685", "U-R8685-2" ] }, { "record_id": "R1029", "passed": true, "expected": [ "S-R1029" ], "actual": [ "S-R1029", "U-R1029-3" ] }, { "record_id": "R4639", "passed": true, "expected": [ "S-R4639" ], "actual": [ "S-R4639" ] } ], "source_sha256": "11a1c3675d05312ebd4561bfa5a187c7834c98e91c70f1dbceb385bbbc982961" } }, { "case_id": "sales-comps-dogwood", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:16:30.228936+00:00", "completed_at": "2026-09-07T03:17:50.097567+00:00", "elapsed_seconds": 79.868, "cost_usd": 0.0828225, "known_partial_cost_usd": 0.0828225, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4322, "completion_tokens": 7742, "total_tokens": 12064, "cost": 0.0828225, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5632, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0828225, "gateway_cost": 0.0828225 }, "finish_reason": "stop", "elapsed_seconds": 79.852, "completed_at": "2026-09-07T03:17:50.096257+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sales-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-dogwood", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5709", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 28000000.0, "expected": 28000000.0, "passed": true }, { "field": "property_size", "actual": 96, "expected": 96, "passed": true }, { "field": "price_per_unit", "actual": 291666.67, "expected": 291666.6666666667, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5709" ], "reported_sources": [ "S-R5709" ] }, { "record_id": "R1935", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 32550000.0, "expected": 32550000.0, "passed": true }, { "field": "property_size", "actual": 103, "expected": 103, "passed": true }, { "field": "price_per_unit", "actual": 316019.42, "expected": 316019.41747572814, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1935", "U-R1935-1" ], "reported_sources": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 32200000.0, "expected": 32200000.000000004, "passed": true }, { "field": "property_size", "actual": 110, "expected": 110, "passed": true }, { "field": "price_per_unit", "actual": 292727.27, "expected": 292727.27272727276, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8685" ], "reported_sources": [ "S-R8685" ] }, { "record_id": "R1029", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 34300000.0, "expected": 34300000.0, "passed": true }, { "field": "property_size", "actual": 118, "expected": 118, "passed": true }, { "field": "price_per_unit", "actual": 290677.97, "expected": 290677.9661016949, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1029" ], "reported_sources": [ "S-R1029" ] }, { "record_id": "R4639", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 36400000.0, "expected": 36400000.0, "passed": true }, { "field": "property_size", "actual": 125, "expected": 125, "passed": true }, { "field": "price_per_unit", "actual": 291200.0, "expected": 291200.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4639" ], "reported_sources": [ "S-R4639" ] } ], "decision_checks": [ { "record_id": "R4639", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3713", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4402", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R5751", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1935", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9886", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3929", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R4855", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5709", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1347", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1328", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7446", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8685", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4477", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4014", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3120", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R1029", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4010", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7792", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R1717", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-dogwood", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:17:41.147568+00:00", "completed_at": "2026-09-07T03:18:36.898849+00:00", "elapsed_seconds": 55.751, "cost_usd": 0.16953, "known_partial_cost_usd": 0.16953, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 5961, "completion_tokens": 5589, "total_tokens": 11550, "cost": 0.16953, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1154, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.16953, "gateway_cost": 0.16953 }, "finish_reason": "stop", "elapsed_seconds": 55.732, "completed_at": "2026-09-07T03:18:36.897238+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sales-comps-dogwood", "expected_target": 5, "expected_decisions": 20, "grade": { "case_id": "sales-comps-dogwood", "task": "sales_comps", "target_count": 5, "available_unique": 7, "returned_count": 5, "eligible_count": 5, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 25, "total": 25 }, "evidence_accuracy": { "passed": 5, "total": 5 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 10, "total": 10 }, "material_errors": [], "selected_checks": [ { "record_id": "R5709", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-78916", "expected": "DOGWOOD-SAL-78916", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 28000000.0, "expected": 28000000.0, "passed": true }, { "field": "property_size", "actual": 96, "expected": 96, "passed": true }, { "field": "price_per_unit", "actual": 291666.67, "expected": 291666.6666666667, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5709" ], "reported_sources": [ "S-R5709", "S-R4477" ] }, { "record_id": "R1935", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-66133", "expected": "DOGWOOD-SAL-66133", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 32550000.0, "expected": 32550000.0, "passed": true }, { "field": "property_size", "actual": 103, "expected": 103, "passed": true }, { "field": "price_per_unit", "actual": 316019.42, "expected": 316019.41747572814, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1935", "U-R1935-1" ], "reported_sources": [ "S-R1935", "U-R1935-1" ] }, { "record_id": "R8685", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-12100", "expected": "DOGWOOD-SAL-12100", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 32200000.0, "expected": 32200000.000000004, "passed": true }, { "field": "property_size", "actual": 110, "expected": 110, "passed": true }, { "field": "price_per_unit", "actual": 292727.27, "expected": 292727.27272727276, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8685" ], "reported_sources": [ "S-R8685" ] }, { "record_id": "R1029", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-64711", "expected": "DOGWOOD-SAL-64711", "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "consideration_usd", "actual": 34300000.0, "expected": 34300000.0, "passed": true }, { "field": "property_size", "actual": 118, "expected": 118, "passed": true }, { "field": "price_per_unit", "actual": 290677.97, "expected": 290677.9661016949, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1029" ], "reported_sources": [ "S-R1029" ] }, { "record_id": "R4639", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SAL-75833", "expected": "DOGWOOD-SAL-75833", "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "consideration_usd", "actual": 36400000.0, "expected": 36400000.0, "passed": true }, { "field": "property_size", "actual": 125, "expected": 125, "passed": true }, { "field": "price_per_unit", "actual": 291200.0, "expected": 291200.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4639" ], "reported_sources": [ "S-R4639" ] } ], "decision_checks": [ { "record_id": "R4639", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3713", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4402", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R5751", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1935", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9886", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3929", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R4855", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R5709", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1347", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R1328", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7446", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R8685", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4477", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4014", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3120", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R1029", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4010", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7792", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R1717", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-elm", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:32:42.590Z", "observed_completed_at": "2026-09-07T02:36:25.466Z", "packet_sha256": "742bfe9a60d3477e30688bf7f0339ea2a20b3f5356e0fe768b4c0ca47e4efdee", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/raleigh_retail_comp_grid.xlsx", "sha256": "372c2c90f65c8d8568e2d2df691da8676da7a64f540434c62b1cd0d4aca1d2cd", "bytes": 8594 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.64498425, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 145.09, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "ec0bae2a-1b7b-4f47-92b8-80d7247baa97", "session_id": "b7d9d90d-307f-46e7-b76a-589fbc9a171c", "started_at": "2026-09-06T22:34:44.193000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.47469425, 0.17029 ], "total_cost_usd": 0.64498425, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 145.09, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sales-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sales-comps-elm", "task": "sales_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R4105", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 14400000.0, "expected": 14400000.0, "passed": true }, { "field": "property_size", "actual": 36000, "expected": 36000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4105" ], "reported_sources": [ "S-R4105" ] }, { "record_id": "R8422", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 16740000.0, "expected": 16740000.0, "passed": true }, { "field": "property_size", "actual": 38700, "expected": 38700, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8422", "U-R8422-1" ], "reported_sources": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 16560000.0, "expected": 16559999.999999998, "passed": true }, { "field": "property_size", "actual": 41400, "expected": 41400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 399.99999999999994, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3397" ], "reported_sources": [ "S-R3397" ] } ], "decision_checks": [ { "record_id": "R8129", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7280", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8422", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7302", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4105", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3794", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3397", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2952", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3577", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3742", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8218", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5078", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4512", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4218", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6058", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2453", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7069", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8810", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R4592", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R6121", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/raleigh_retail_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions", "Summary" ], "table_sheet": "Comp Grid", "original_record_order": [ "R4105", "R8422", "R3397" ], "answer_record_order": [ "R4105", "R8422", "R3397" ], "field_checks": [ { "record_id": "R4105", "field": "entity_id", "actual": "ELM-SAL-91902", "original": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "record_id": "R4105", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R4105", "field": "consideration_usd", "actual": 14400000, "original": 14400000, "expected": 14400000.0, "passed": true }, { "record_id": "R4105", "field": "property_size", "actual": 36000, "original": 36000, "expected": 36000, "passed": true }, { "record_id": "R4105", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R8422", "field": "entity_id", "actual": "ELM-SAL-64090", "original": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "record_id": "R8422", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8422", "field": "consideration_usd", "actual": 16740000, "original": 16740000, "expected": 16740000.0, "passed": true }, { "record_id": "R8422", "field": "property_size", "actual": 38700, "original": 38700, "expected": 38700, "passed": true }, { "record_id": "R8422", "field": "price_per_unit", "actual": 432.56, "original": 432.56, "expected": 432.56, "passed": true }, { "record_id": "R3397", "field": "entity_id", "actual": "ELM-SAL-96210", "original": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "record_id": "R3397", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R3397", "field": "consideration_usd", "actual": 16560000, "original": 16560000, "expected": 16560000.0, "passed": true }, { "record_id": "R3397", "field": "property_size", "actual": 41400, "original": 41400, "expected": 41400, "passed": true }, { "record_id": "R3397", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R4105", "passed": true, "expected": [ "S-R4105" ], "actual": [ "S-R4105" ] }, { "record_id": "R8422", "passed": true, "expected": [ "S-R8422", "U-R8422-1" ], "actual": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "passed": true, "expected": [ "S-R3397" ], "actual": [ "S-R3397" ] } ], "source_sha256": "372c2c90f65c8d8568e2d2df691da8676da7a64f540434c62b1cd0d4aca1d2cd" } }, { "case_id": "sales-comps-elm", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:17:50.098570+00:00", "completed_at": "2026-09-07T03:21:09.543955+00:00", "elapsed_seconds": 199.445, "cost_usd": 0.225564, "known_partial_cost_usd": 0.225564, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 4, "tool_calls": 4, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5350, "completion_tokens": 2377, "total_tokens": 7727, "cost": 0.0304575, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2304, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0304575, "gateway_cost": 0.0304575 }, "finish_reason": "tool_calls", "elapsed_seconds": 20.688, "completed_at": "2026-09-07T03:18:10.815002+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5399, "completion_tokens": 5361, "total_tokens": 10760, "cost": 0.05445475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5248, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4736, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05445475, "gateway_cost": 0.05445475 }, "finish_reason": "tool_calls", "elapsed_seconds": 59.667, "completed_at": "2026-09-07T03:19:10.487600+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6012, "completion_tokens": 5929, "total_tokens": 11941, "cost": 0.060757, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5376, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5888, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.060757, "gateway_cost": 0.060757 }, "finish_reason": "tool_calls", "elapsed_seconds": 51.143, "completed_at": "2026-09-07T03:20:02.138401+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6055, "completion_tokens": 7895, "total_tokens": 13950, "cost": 0.07989475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5888, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6464, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07989475, "gateway_cost": 0.07989475 }, "finish_reason": "tool_calls", "elapsed_seconds": 67.399, "completed_at": "2026-09-07T03:21:09.540646+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "bab688a91681a04394f315d45846c848742e5a827db240b2f52464f7b023332d", "bytes": 4095 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sales-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sales-comps-elm", "task": "sales_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R4105", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 14400000, "expected": 14400000.0, "passed": true }, { "field": "property_size", "actual": 36000, "expected": 36000, "passed": true }, { "field": "price_per_unit", "actual": 400, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4105" ], "reported_sources": [ "S-R4105" ] }, { "record_id": "R8422", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 16740000, "expected": 16740000.0, "passed": true }, { "field": "property_size", "actual": 38700, "expected": 38700, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8422", "U-R8422-1" ], "reported_sources": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 16560000, "expected": 16559999.999999998, "passed": true }, { "field": "property_size", "actual": 41400, "expected": 41400, "passed": true }, { "field": "price_per_unit", "actual": 400, "expected": 399.99999999999994, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3397" ], "reported_sources": [ "S-R3397" ] } ], "decision_checks": [ { "record_id": "R8129", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7280", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8422", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7302", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4105", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3794", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3397", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2952", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3577", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3742", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8218", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5078", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4512", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4218", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6058", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2453", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7069", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8810", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R4592", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R6121", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [ "R4105", "R8422", "R3397" ], "answer_record_order": [ "R4105", "R8422", "R3397" ], "field_checks": [ { "record_id": "R4105", "field": "entity_id", "actual": "ELM-SAL-91902", "original": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "record_id": "R4105", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R4105", "field": "consideration_usd", "actual": 14400000, "original": 14400000, "expected": 14400000, "passed": true }, { "record_id": "R4105", "field": "property_size", "actual": 36000, "original": 36000, "expected": 36000, "passed": true }, { "record_id": "R4105", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400, "passed": true }, { "record_id": "R8422", "field": "entity_id", "actual": "ELM-SAL-64090", "original": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "record_id": "R8422", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8422", "field": "consideration_usd", "actual": 16740000, "original": 16740000, "expected": 16740000, "passed": true }, { "record_id": "R8422", "field": "property_size", "actual": 38700, "original": 38700, "expected": 38700, "passed": true }, { "record_id": "R8422", "field": "price_per_unit", "actual": 432.5581395348837, "original": 432.5581395348837, "expected": 432.56, "passed": true }, { "record_id": "R3397", "field": "entity_id", "actual": "ELM-SAL-96210", "original": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "record_id": "R3397", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R3397", "field": "consideration_usd", "actual": 16560000, "original": 16560000, "expected": 16560000, "passed": true }, { "record_id": "R3397", "field": "property_size", "actual": 41400, "original": 41400, "expected": 41400, "passed": true }, { "record_id": "R3397", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R4105", "passed": true, "expected": [ "S-R4105" ], "actual": [ "S-R4105" ] }, { "record_id": "R8422", "passed": true, "expected": [ "S-R8422", "U-R8422-1" ], "actual": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "passed": true, "expected": [ "S-R3397" ], "actual": [ "S-R3397" ] } ], "source_sha256": "bab688a91681a04394f315d45846c848742e5a827db240b2f52464f7b023332d" } }, { "case_id": "sales-comps-elm", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:18:36.899735+00:00", "completed_at": "2026-09-07T03:20:05.957610+00:00", "elapsed_seconds": 89.058, "cost_usd": 0.383565, "known_partial_cost_usd": 0.383565, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8525, "completion_tokens": 1142, "total_tokens": 9667, "cost": 0.071175, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1011, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.071175, "gateway_cost": 0.071175 }, "finish_reason": "tool_calls", "elapsed_seconds": 13.257, "completed_at": "2026-09-07T03:18:50.174941+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8713, "completion_tokens": 4761, "total_tokens": 13474, "cost": 0.16259, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 818, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.16259, "gateway_cost": 0.16259 }, "finish_reason": "tool_calls", "elapsed_seconds": 42.162, "completed_at": "2026-09-07T03:19:32.340515+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13220, "completion_tokens": 3348, "total_tokens": 16568, "cost": 0.1498, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 12, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.1498, "gateway_cost": 0.1498 }, "finish_reason": "stop", "elapsed_seconds": 33.094, "completed_at": "2026-09-07T03:20:05.955964+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "743a378ec650e68c65cf0799208b0fff3dc6922d559b6e464b92f1af224d5d2c", "bytes": 6301 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sales-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sales-comps-elm", "task": "sales_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R4105", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 14400000, "expected": 14400000.0, "passed": true }, { "field": "property_size", "actual": 36000, "expected": 36000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4105" ], "reported_sources": [ "S-R4105", "S-R3577" ] }, { "record_id": "R8422", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 16740000, "expected": 16740000.0, "passed": true }, { "field": "property_size", "actual": 38700, "expected": 38700, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8422", "U-R8422-1" ], "reported_sources": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 16560000, "expected": 16559999.999999998, "passed": true }, { "field": "property_size", "actual": 41400, "expected": 41400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 399.99999999999994, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3397" ], "reported_sources": [ "S-R3397" ] } ], "decision_checks": [ { "record_id": "R8129", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7280", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8422", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7302", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4105", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3794", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3397", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2952", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3577", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3742", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8218", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5078", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4512", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4218", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6058", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2453", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7069", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8810", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R4592", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R6121", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Excluded" ], "table_sheet": "Comp Grid", "original_record_order": [ "R4105", "R8422", "R3397" ], "answer_record_order": [ "R4105", "R8422", "R3397" ], "field_checks": [ { "record_id": "R4105", "field": "entity_id", "actual": "ELM-SAL-91902", "original": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "record_id": "R4105", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R4105", "field": "consideration_usd", "actual": 14400000, "original": 14400000, "expected": 14400000, "passed": true }, { "record_id": "R4105", "field": "property_size", "actual": 36000, "original": 36000, "expected": 36000, "passed": true }, { "record_id": "R4105", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true }, { "record_id": "R8422", "field": "entity_id", "actual": "ELM-SAL-64090", "original": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "record_id": "R8422", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8422", "field": "consideration_usd", "actual": 16740000, "original": 16740000, "expected": 16740000, "passed": true }, { "record_id": "R8422", "field": "property_size", "actual": 38700, "original": 38700, "expected": 38700, "passed": true }, { "record_id": "R8422", "field": "price_per_unit", "actual": 432.5581395348837, "original": 432.5581395348837, "expected": 432.56, "passed": true }, { "record_id": "R3397", "field": "entity_id", "actual": "ELM-SAL-96210", "original": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "record_id": "R3397", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R3397", "field": "consideration_usd", "actual": 16560000, "original": 16560000, "expected": 16560000, "passed": true }, { "record_id": "R3397", "field": "property_size", "actual": 41400, "original": 41400, "expected": 41400, "passed": true }, { "record_id": "R3397", "field": "price_per_unit", "actual": 400, "original": 400, "expected": 400.0, "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R4105", "passed": true, "expected": [ "S-R3577", "S-R4105" ], "actual": [ "S-R3577", "S-R4105" ] }, { "record_id": "R8422", "passed": true, "expected": [ "S-R8422", "U-R8422-1" ], "actual": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "passed": true, "expected": [ "S-R3397" ], "actual": [ "S-R3397", "U-R3397-2" ] } ], "source_sha256": "743a378ec650e68c65cf0799208b0fff3dc6922d559b6e464b92f1af224d5d2c" } }, { "case_id": "sales-comps-elm", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:18:38.440448+00:00", "completed_at": "2026-09-07T03:20:41.756437+00:00", "elapsed_seconds": 123.316, "cost_usd": 0.090125, "known_partial_cost_usd": 0.090125, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4604, "completion_tokens": 8437, "total_tokens": 13041, "cost": 0.090125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6592, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.090125, "gateway_cost": 0.090125 }, "finish_reason": "stop", "elapsed_seconds": 123.297, "completed_at": "2026-09-07T03:20:41.754902+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sales-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sales-comps-elm", "task": "sales_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R4105", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 14400000.0, "expected": 14400000.0, "passed": true }, { "field": "property_size", "actual": 36000, "expected": 36000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4105" ], "reported_sources": [ "S-R4105" ] }, { "record_id": "R8422", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 16740000.0, "expected": 16740000.0, "passed": true }, { "field": "property_size", "actual": 38700, "expected": 38700, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8422", "U-R8422-1" ], "reported_sources": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 16560000.0, "expected": 16559999.999999998, "passed": true }, { "field": "property_size", "actual": 41400, "expected": 41400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 399.99999999999994, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3397" ], "reported_sources": [ "S-R3397" ] } ], "decision_checks": [ { "record_id": "R8129", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7280", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8422", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7302", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4105", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3794", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3397", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2952", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3577", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3742", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8218", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5078", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4512", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4218", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6058", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2453", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7069", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8810", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R4592", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R6121", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-elm", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:20:05.959853+00:00", "completed_at": "2026-09-07T03:20:53.138605+00:00", "elapsed_seconds": 47.179, "cost_usd": 0.150125, "known_partial_cost_usd": 0.150125, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 6240, "completion_tokens": 4757, "total_tokens": 10997, "cost": 0.150125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 896, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.150125, "gateway_cost": 0.150125 }, "finish_reason": "stop", "elapsed_seconds": 47.16, "completed_at": "2026-09-07T03:20:53.136951+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sales-comps-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sales-comps-elm", "task": "sales_comps", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 15, "total": 15 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R4105", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-91902", "expected": "ELM-SAL-91902", "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "consideration_usd", "actual": 14400000, "expected": 14400000.0, "passed": true }, { "field": "property_size", "actual": 36000, "expected": 36000, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 400.0, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4105" ], "reported_sources": [ "S-R4105" ] }, { "record_id": "R8422", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-64090", "expected": "ELM-SAL-64090", "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "consideration_usd", "actual": 16740000, "expected": 16740000.0, "passed": true }, { "field": "property_size", "actual": 38700, "expected": 38700, "passed": true }, { "field": "price_per_unit", "actual": 432.56, "expected": 432.5581395348837, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8422", "U-R8422-1" ], "reported_sources": [ "S-R8422", "U-R8422-1" ] }, { "record_id": "R3397", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SAL-96210", "expected": "ELM-SAL-96210", "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "consideration_usd", "actual": 16560000, "expected": 16559999.999999998, "passed": true }, { "field": "property_size", "actual": 41400, "expected": 41400, "passed": true }, { "field": "price_per_unit", "actual": 400.0, "expected": 399.99999999999994, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3397" ], "reported_sources": [ "S-R3397" ] } ], "decision_checks": [ { "record_id": "R8129", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R7280", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8422", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7302", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4105", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3794", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3397", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2952", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R3577", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3742", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8218", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R5078", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R4512", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R4218", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R6058", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2453", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7069", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8810", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R4592", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R6121", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-fir", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:32:45.968Z", "observed_completed_at": "2026-09-07T02:36:25.879Z", "packet_sha256": "bc407b37c4cb6f239478b886137ee2f0d9efe551c95ede06b04bf62e5dcc8ff0", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/dallas_industrial_comp_grid.xlsx", "sha256": "0c9fd21a804760cbbd4a72233b60d5537cf3cbd1defeb0eadb232bf3ff1b7dd0", "bytes": 7125 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.3223155, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 79.04, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "981c19ec-d294-4a9f-9983-61f80fe73dfd", "session_id": "5f115b9d-4ca8-4e9a-8b54-945d438b4e5a", "started_at": "2026-09-06T22:33:43.168000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.221053, 0.1012625 ], "total_cost_usd": 0.3223155, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 79.04, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sales-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sales-comps-fir", "task": "sales_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R8490", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R8114", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R8252", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2607", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2188", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7726", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2939", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6997", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9929", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9726", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R8402", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2942", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3099", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3387", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R8382", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7274", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8181", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R4189", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/dallas_industrial_comp_grid.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Candidate Decisions" ], "table_sheet": "Comp Grid", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "0c9fd21a804760cbbd4a72233b60d5537cf3cbd1defeb0eadb232bf3ff1b7dd0" } }, { "case_id": "sales-comps-fir", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:20:41.757541+00:00", "completed_at": "2026-09-07T03:22:33.512100+00:00", "elapsed_seconds": 111.754, "cost_usd": 0.12877675, "known_partial_cost_usd": 0.12877675, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5563, "completion_tokens": 5448, "total_tokens": 11011, "cost": 0.06143375, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5184, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06143375, "gateway_cost": 0.06143375 }, "finish_reason": "tool_calls", "elapsed_seconds": 49.761, "completed_at": "2026-09-07T03:21:31.538021+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5868, "completion_tokens": 6620, "total_tokens": 12488, "cost": 0.067343, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 5504, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5440, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.067343, "gateway_cost": 0.067343 }, "finish_reason": "tool_calls", "elapsed_seconds": 61.465, "completed_at": "2026-09-07T03:22:33.510169+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "76324f151b25b90f8dab0bafe023816c9a029b2a60339e723f922baacd7eb308", "bytes": 3926 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sales-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sales-comps-fir", "task": "sales_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R8490", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R8114", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R8252", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2607", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2188", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7726", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2939", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6997", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9929", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9726", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R8402", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2942", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3099", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3387", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R8382", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7274", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8181", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R4189", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid" ], "table_sheet": "Comp Grid", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "76324f151b25b90f8dab0bafe023816c9a029b2a60339e723f922baacd7eb308" } }, { "case_id": "sales-comps-fir", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:20:53.139497+00:00", "completed_at": "2026-09-07T03:22:40.136554+00:00", "elapsed_seconds": 106.997, "cost_usd": 0.389185, "known_partial_cost_usd": 0.389185, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8820, "completion_tokens": 5722, "total_tokens": 14542, "cost": 0.18715, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1484, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.18715, "gateway_cost": 0.18715 }, "finish_reason": "tool_calls", "elapsed_seconds": 53.17, "completed_at": "2026-09-07T03:21:46.326092+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 13247, "completion_tokens": 5432, "total_tokens": 18679, "cost": 0.202035, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 409, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.202035, "gateway_cost": 0.202035 }, "finish_reason": "tool_calls", "elapsed_seconds": 53.268, "completed_at": "2026-09-07T03:22:40.134254+00:00" } ], "artifacts": [ { "path": "artifacts/underwriting-v1.xlsx", "sha256": "d4f96dc3b8139395639769a2922bd9ba066f7a192d619f8f146931fab0681ac9", "bytes": 6981 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sales-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sales-comps-fir", "task": "sales_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R8490", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R8114", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R8252", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2607", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2188", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7726", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2939", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6997", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9929", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9726", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R8402", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2942", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3099", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3387", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R8382", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7274", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8181", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R4189", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".xlsx", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/underwriting-v1.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "Comp Grid", "Screening" ], "table_sheet": "Comp Grid", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "d4f96dc3b8139395639769a2922bd9ba066f7a192d619f8f146931fab0681ac9" } }, { "case_id": "sales-comps-fir", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:21:09.545382+00:00", "completed_at": "2026-09-07T03:23:57.211480+00:00", "elapsed_seconds": 167.666, "cost_usd": 0.07505125, "known_partial_cost_usd": 0.07505125, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 4817, "completion_tokens": 6903, "total_tokens": 11720, "cost": 0.07505125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5440, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07505125, "gateway_cost": 0.07505125 }, "finish_reason": "stop", "elapsed_seconds": 167.647, "completed_at": "2026-09-07T03:23:57.210030+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sales-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sales-comps-fir", "task": "sales_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R8490", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R8114", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R8252", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2607", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2188", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7726", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2939", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6997", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9929", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9726", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R8402", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2942", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3099", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3387", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R8382", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7274", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8181", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R4189", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sales-comps-fir", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:22:33.513008+00:00", "completed_at": "2026-09-07T03:23:14.327307+00:00", "elapsed_seconds": 40.814, "cost_usd": 0.12835, "known_partial_cost_usd": 0.12835, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 6535, "completion_tokens": 3827, "total_tokens": 10362, "cost": 0.12835, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 882, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.12835, "gateway_cost": 0.12835 }, "finish_reason": "stop", "elapsed_seconds": 40.795, "completed_at": "2026-09-07T03:23:14.325478+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sales_comps", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sales-comps-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sales-comps-fir", "task": "sales_comps", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R8490", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside size range" ] }, { "record_id": "R8114", "actual": false, "expected": false, "passed": true, "reasons": [ "No positive verified price" ] }, { "record_id": "R8252", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2607", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2188", "actual": false, "expected": false, "passed": true, "reasons": [ "scope is portfolio_unallocated" ] }, { "record_id": "R7726", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2939", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside radius" ] }, { "record_id": "R6997", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9929", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9726", "actual": false, "expected": false, "passed": true, "reasons": [ "Related-party transaction" ] }, { "record_id": "R8402", "actual": false, "expected": false, "passed": true, "reasons": [ "interest is minority_equity_interest" ] }, { "record_id": "R2942", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3099", "actual": false, "expected": false, "passed": true, "reasons": [ "status is listing" ] }, { "record_id": "R3387", "actual": false, "expected": false, "passed": true, "reasons": [ "size_unit is acre" ] }, { "record_id": "R8382", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7274", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8181", "actual": false, "expected": false, "passed": true, "reasons": [ "condition is redevelopment" ] }, { "record_id": "R4189", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-alder", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:20:57.445Z", "observed_completed_at": "2026-09-07T02:26:31.471Z", "packet_sha256": "59068690a67d0cf69e8b22f9f50b1bb33ff357d1a5412b37f662f3d6081ae51e", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/alder_charlotte_mf_prospects.xlsx", "sha256": "86857a125418a970866361d7ea8cfd967795b5a937f285f9b938599427cf7971", "bytes": 9443 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.544048, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 121.69, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "d08304ae-33a0-47ff-8d12-538b81bd708a", "session_id": "c76551e3-bdbd-415c-90df-22230f6a21e9", "started_at": "2026-09-06T22:22:21.140000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.3640655, 0.1799825 ], "total_cost_usd": 0.544048, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 121.69, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sponsor-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-alder", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R9772", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R9772", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-86867", "expected": "ALDER-SPO-86867", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 18 Capital", "expected": "Alder Grove 18 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder18", "expected": "Morgan Alder18", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person18@sponsor-leads-alder.example", "expected": "person18@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9772" ], "reported_sources": [ "S-R9772" ] }, { "record_id": "R3168", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "field": "firm_name", "actual": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 33000000, "expected": 33000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3168" ], "reported_sources": [ "S-R3168" ] }, { "record_id": "R4787", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "field": "firm_name", "actual": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 31200000, "expected": 31200000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4787" ], "reported_sources": [ "S-R4787" ] }, { "record_id": "R4752", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4752" ], "reported_sources": [ "S-R4752" ] }, { "record_id": "R2273", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "field": "firm_name", "actual": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 27600000, "expected": 27600000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2273" ], "reported_sources": [ "S-R2273" ] }, { "record_id": "R1117", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "field": "firm_name", "actual": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 25800000, "expected": 25800000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1117", "U-R1117-1" ], "reported_sources": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R5723", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 24000000, "expected": 24000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R5723" ], "reported_sources": [ "S-R5723", "S-R7805" ] } ], "decision_checks": [ { "record_id": "R9772", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4022", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1654", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5723", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1117", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4787", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3335", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R1421", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2273", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1078", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R7756", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2889", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4752", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4993", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R3168", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9496", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2033", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7805", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4644", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": false }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/alder_charlotte_mf_prospects.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "Requested .csv; delivered .xlsx" ], "sheets": [ "Prospects", "Candidate Decisions", "Notes" ], "table_sheet": "Prospects", "original_record_order": [ "R9772", "R3168", "R4787", "R4752", "R2273", "R1117", "R5723" ], "answer_record_order": [ "R9772", "R3168", "R4787", "R4752", "R2273", "R1117", "R5723" ], "field_checks": [ { "record_id": "R9772", "field": "entity_id", "actual": "ALDER-SPO-86867", "original": "ALDER-SPO-86867", "expected": "ALDER-SPO-86867", "passed": true }, { "record_id": "R9772", "field": "firm_name", "actual": "Alder Grove 18 Capital", "original": "Alder Grove 18 Capital", "expected": "Alder Grove 18 Capital", "passed": true }, { "record_id": "R9772", "field": "transaction_size_usd", "actual": 29400000, "original": 29400000, "expected": 29400000, "passed": true }, { "record_id": "R9772", "field": "event_date", "actual": "2026-07-22", "original": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "record_id": "R9772", "field": "contact_name", "actual": "Morgan Alder18", "original": "Morgan Alder18", "expected": "Morgan Alder18", "passed": true }, { "record_id": "R9772", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R9772", "field": "contact_email", "actual": "person18@sponsor-leads-alder.example", "original": "person18@sponsor-leads-alder.example", "expected": "person18@sponsor-leads-alder.example", "passed": true }, { "record_id": "R3168", "field": "entity_id", "actual": "ALDER-SPO-67037", "original": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "record_id": "R3168", "field": "firm_name", "actual": "Alder Square 6 Capital", "original": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "record_id": "R3168", "field": "transaction_size_usd", "actual": 33000000, "original": 33000000, "expected": 33000000, "passed": true }, { "record_id": "R3168", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R3168", "field": "contact_name", "actual": "Morgan Alder6", "original": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "record_id": "R3168", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R3168", "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "original": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true }, { "record_id": "R4787", "field": "entity_id", "actual": "ALDER-SPO-19257", "original": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "record_id": "R4787", "field": "firm_name", "actual": "Alder Square 5 Capital", "original": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "record_id": "R4787", "field": "transaction_size_usd", "actual": 31200000, "original": 31200000, "expected": 31200000, "passed": true }, { "record_id": "R4787", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R4787", "field": "contact_name", "actual": "Alex Alder5", "original": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "record_id": "R4787", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4787", "field": "contact_email", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R4752", "field": "entity_id", "actual": "ALDER-SPO-13273", "original": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "record_id": "R4752", "field": "firm_name", "actual": "Alder Grove 4 Capital", "original": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "record_id": "R4752", "field": "transaction_size_usd", "actual": 29400000, "original": 29400000, "expected": 29400000, "passed": true }, { "record_id": "R4752", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4752", "field": "contact_name", "actual": "Taylor Alder4", "original": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "record_id": "R4752", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R4752", "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "original": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true }, { "record_id": "R2273", "field": "entity_id", "actual": "ALDER-SPO-21979", "original": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "record_id": "R2273", "field": "firm_name", "actual": "Alder Park 3 Capital", "original": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "record_id": "R2273", "field": "transaction_size_usd", "actual": 27600000, "original": 27600000, "expected": 27600000, "passed": true }, { "record_id": "R2273", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2273", "field": "contact_name", "actual": "Jordan Alder3", "original": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "record_id": "R2273", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2273", "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "original": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true }, { "record_id": "R1117", "field": "entity_id", "actual": "ALDER-SPO-82903", "original": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "record_id": "R1117", "field": "firm_name", "actual": "Alder Park 2 Capital", "original": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "record_id": "R1117", "field": "transaction_size_usd", "actual": 25800000, "original": 25800000, "expected": 25800000, "passed": true }, { "record_id": "R1117", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1117", "field": "contact_name", "actual": "Drew Alder", "original": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "record_id": "R1117", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1117", "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "original": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true }, { "record_id": "R5723", "field": "entity_id", "actual": "ALDER-SPO-80621", "original": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "record_id": "R5723", "field": "firm_name", "actual": "Alder Grove 1 Capital", "original": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "record_id": "R5723", "field": "transaction_size_usd", "actual": 24000000, "original": 24000000, "expected": 24000000, "passed": true }, { "record_id": "R5723", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R5723", "field": "contact_name", "actual": "Alex Alder1", "original": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "record_id": "R5723", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R5723", "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "original": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R9772", "passed": true, "expected": [ "S-R9772" ], "actual": [ "S-R9772" ] }, { "record_id": "R3168", "passed": true, "expected": [ "S-R3168" ], "actual": [ "S-R3168" ] }, { "record_id": "R4787", "passed": true, "expected": [ "S-R4787" ], "actual": [ "S-R4787" ] }, { "record_id": "R4752", "passed": true, "expected": [ "S-R4752" ], "actual": [ "S-R4752", "U-R4752-3" ] }, { "record_id": "R2273", "passed": true, "expected": [ "S-R2273" ], "actual": [ "S-R2273", "U-R2273-2" ] }, { "record_id": "R1117", "passed": true, "expected": [ "S-R1117", "U-R1117-1" ], "actual": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R5723", "passed": true, "expected": [ "S-R5723", "S-R7805" ], "actual": [ "S-R5723", "S-R7805" ] } ], "source_sha256": "86857a125418a970866361d7ea8cfd967795b5a937f285f9b938599427cf7971" } }, { "case_id": "sponsor-leads-alder", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:21:56.674697+00:00", "completed_at": "2026-09-07T02:26:16.934049+00:00", "elapsed_seconds": 260.258, "cost_usd": 0.24883899999999998, "known_partial_cost_usd": 0.24883899999999998, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6210, "completion_tokens": 7112, "total_tokens": 13322, "cost": 0.0788825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6656, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0788825, "gateway_cost": 0.0788825 }, "finish_reason": "tool_calls", "elapsed_seconds": 90.057, "completed_at": "2026-09-07T02:23:26.751382+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6668, "completion_tokens": 7121, "total_tokens": 13789, "cost": 0.072633, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6144, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6656, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.072633, "gateway_cost": 0.072633 }, "finish_reason": "tool_calls", "elapsed_seconds": 69.039, "completed_at": "2026-09-07T02:24:35.794513+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7126, "completion_tokens": 9576, "total_tokens": 16702, "cost": 0.0973235, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6528, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7296, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0973235, "gateway_cost": 0.0973235 }, "finish_reason": "tool_calls", "elapsed_seconds": 101.133, "completed_at": "2026-09-07T02:26:16.931727+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "8535ac3c17f3b1e020a51c40ac3fa124c8a4bb929d66cc88c91dedf5a5b06072", "bytes": 1033 }, { "path": "artifacts/prospects-v2.csv", "sha256": "8535ac3c17f3b1e020a51c40ac3fa124c8a4bb929d66cc88c91dedf5a5b06072", "bytes": 1033 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-alder", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R9772", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R9772", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-86867", "expected": "ALDER-SPO-86867", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 18 Capital", "expected": "Alder Grove 18 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder18", "expected": "Morgan Alder18", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person18@sponsor-leads-alder.example", "expected": "person18@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9772" ], "reported_sources": [ "S-R9772" ] }, { "record_id": "R3168", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "field": "firm_name", "actual": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 33000000, "expected": 33000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3168" ], "reported_sources": [ "S-R3168" ] }, { "record_id": "R4787", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "field": "firm_name", "actual": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 31200000, "expected": 31200000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4787" ], "reported_sources": [ "S-R4787" ] }, { "record_id": "R4752", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4752" ], "reported_sources": [ "S-R4752" ] }, { "record_id": "R2273", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "field": "firm_name", "actual": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 27600000, "expected": 27600000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2273" ], "reported_sources": [ "S-R2273" ] }, { "record_id": "R1117", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "field": "firm_name", "actual": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 25800000, "expected": 25800000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1117", "U-R1117-1" ], "reported_sources": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R7756", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34800000, "expected": 34800000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7756", "S-R7805" ], "reported_sources": [ "S-R7756", "S-R7805" ] } ], "decision_checks": [ { "record_id": "R9772", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4022", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1654", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5723", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1117", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4787", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3335", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R1421", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2273", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1078", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R7756", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2889", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4752", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4993", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R3168", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9496", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2033", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7805", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4644", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v2.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R4787 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R9772", "R3168", "R4787", "R4752", "R2273", "R1117", "R7756" ], "answer_record_order": [ "R9772", "R3168", "R4787", "R4752", "R2273", "R1117", "R7756" ], "field_checks": [ { "record_id": "R9772", "field": "entity_id", "actual": "ALDER-SPO-86867", "original": "ALDER-SPO-86867", "expected": "ALDER-SPO-86867", "passed": true }, { "record_id": "R9772", "field": "firm_name", "actual": "Alder Grove 18 Capital", "original": "Alder Grove 18 Capital", "expected": "Alder Grove 18 Capital", "passed": true }, { "record_id": "R9772", "field": "transaction_size_usd", "actual": "29400000", "original": "29400000", "expected": 29400000, "passed": true }, { "record_id": "R9772", "field": "event_date", "actual": "2026-07-22", "original": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "record_id": "R9772", "field": "contact_name", "actual": "Morgan Alder18", "original": "Morgan Alder18", "expected": "Morgan Alder18", "passed": true }, { "record_id": "R9772", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R9772", "field": "contact_email", "actual": "person18@sponsor-leads-alder.example", "original": "person18@sponsor-leads-alder.example", "expected": "person18@sponsor-leads-alder.example", "passed": true }, { "record_id": "R3168", "field": "entity_id", "actual": "ALDER-SPO-67037", "original": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "record_id": "R3168", "field": "firm_name", "actual": "Alder Square 6 Capital", "original": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "record_id": "R3168", "field": "transaction_size_usd", "actual": "33000000", "original": "33000000", "expected": 33000000, "passed": true }, { "record_id": "R3168", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R3168", "field": "contact_name", "actual": "Morgan Alder6", "original": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "record_id": "R3168", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R3168", "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "original": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true }, { "record_id": "R4787", "field": "entity_id", "actual": "ALDER-SPO-19257", "original": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "record_id": "R4787", "field": "firm_name", "actual": "Alder Square 5 Capital", "original": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "record_id": "R4787", "field": "transaction_size_usd", "actual": "31200000", "original": "31200000", "expected": 31200000, "passed": true }, { "record_id": "R4787", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R4787", "field": "contact_name", "actual": "Alex Alder5", "original": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "record_id": "R4787", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4787", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-alder/turn-02/tool-01.json" }, { "record_id": "R4752", "field": "entity_id", "actual": "ALDER-SPO-13273", "original": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "record_id": "R4752", "field": "firm_name", "actual": "Alder Grove 4 Capital", "original": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "record_id": "R4752", "field": "transaction_size_usd", "actual": "29400000", "original": "29400000", "expected": 29400000, "passed": true }, { "record_id": "R4752", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4752", "field": "contact_name", "actual": "Taylor Alder4", "original": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "record_id": "R4752", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R4752", "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "original": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true }, { "record_id": "R2273", "field": "entity_id", "actual": "ALDER-SPO-21979", "original": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "record_id": "R2273", "field": "firm_name", "actual": "Alder Park 3 Capital", "original": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "record_id": "R2273", "field": "transaction_size_usd", "actual": "27600000", "original": "27600000", "expected": 27600000, "passed": true }, { "record_id": "R2273", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2273", "field": "contact_name", "actual": "Jordan Alder3", "original": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "record_id": "R2273", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2273", "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "original": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true }, { "record_id": "R1117", "field": "entity_id", "actual": "ALDER-SPO-82903", "original": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "record_id": "R1117", "field": "firm_name", "actual": "Alder Park 2 Capital", "original": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "record_id": "R1117", "field": "transaction_size_usd", "actual": "25800000", "original": "25800000", "expected": 25800000, "passed": true }, { "record_id": "R1117", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1117", "field": "contact_name", "actual": "Drew Alder", "original": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "record_id": "R1117", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1117", "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "original": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true }, { "record_id": "R7756", "field": "entity_id", "actual": "ALDER-SPO-80621", "original": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "record_id": "R7756", "field": "firm_name", "actual": "Alder Grove 1 Capital", "original": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "record_id": "R7756", "field": "transaction_size_usd", "actual": "34800000", "original": "34800000", "expected": 34800000, "passed": true }, { "record_id": "R7756", "field": "event_date", "actual": "2026-02-11", "original": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "record_id": "R7756", "field": "contact_name", "actual": "Alex Alder1", "original": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "record_id": "R7756", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R7756", "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "original": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R9772", "passed": true, "expected": [ "S-R9772" ], "actual": [ "S-R9772" ] }, { "record_id": "R3168", "passed": true, "expected": [ "S-R3168" ], "actual": [ "S-R3168" ] }, { "record_id": "R4787", "passed": true, "expected": [ "S-R4787" ], "actual": [ "S-R4787" ] }, { "record_id": "R4752", "passed": true, "expected": [ "S-R4752" ], "actual": [ "S-R4752" ] }, { "record_id": "R2273", "passed": true, "expected": [ "S-R2273" ], "actual": [ "S-R2273" ] }, { "record_id": "R1117", "passed": true, "expected": [ "S-R1117", "U-R1117-1" ], "actual": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R7756", "passed": true, "expected": [ "S-R7756", "S-R7805" ], "actual": [ "S-R7756", "S-R7805" ] } ], "source_sha256": "8535ac3c17f3b1e020a51c40ac3fa124c8a4bb929d66cc88c91dedf5a5b06072" } }, { "case_id": "sponsor-leads-alder", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:22:33.390700+00:00", "completed_at": "2026-09-07T02:23:53.471997+00:00", "elapsed_seconds": 80.081, "cost_usd": 0.30266, "known_partial_cost_usd": 0.30266, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 1, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10235, "completion_tokens": 2695, "total_tokens": 12930, "cost": 0.11855, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 2021, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11855, "gateway_cost": 0.11855 }, "finish_reason": "tool_calls", "elapsed_seconds": 29.605, "completed_at": "2026-09-07T02:23:03.006202+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10987, "completion_tokens": 5167, "total_tokens": 16154, "cost": 0.18411, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 660, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.18411, "gateway_cost": 0.18411 }, "finish_reason": "stop", "elapsed_seconds": 50.458, "completed_at": "2026-09-07T02:23:53.469237+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "fed84c21a6038dd9f19f674ceb897bc58aca6ee910f5a3d106a509f6caf9096d", "bytes": 1004 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-alder", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3168", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "field": "firm_name", "actual": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 33000000, "expected": 33000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3168" ], "reported_sources": [ "S-R3168" ] }, { "record_id": "R4787", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "field": "firm_name", "actual": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 31200000, "expected": 31200000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4787" ], "reported_sources": [ "S-R4787" ] }, { "record_id": "R4752", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4752" ], "reported_sources": [ "S-R4752" ] }, { "record_id": "R2273", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "field": "firm_name", "actual": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 27600000, "expected": 27600000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2273" ], "reported_sources": [ "S-R2273" ] }, { "record_id": "R1117", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "field": "firm_name", "actual": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 25800000, "expected": 25800000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1117", "U-R1117-1" ], "reported_sources": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R7805", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 24000000, "expected": 24000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7805" ], "reported_sources": [ "S-R7805", "S-R5723", "S-R7756" ] } ], "decision_checks": [ { "record_id": "R9772", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4022", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1654", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5723", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1117", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4787", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3335", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R1421", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2273", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1078", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R7756", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2889", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4752", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4993", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R3168", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9496", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2033", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7805", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4644", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R4787 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R3168", "R4787", "R4752", "R2273", "R1117", "R7805" ], "answer_record_order": [ "R3168", "R4787", "R4752", "R2273", "R1117", "R7805" ], "field_checks": [ { "record_id": "R3168", "field": "entity_id", "actual": "ALDER-SPO-67037", "original": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "record_id": "R3168", "field": "firm_name", "actual": "Alder Square 6 Capital", "original": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "record_id": "R3168", "field": "transaction_size_usd", "actual": "33000000", "original": "33000000", "expected": 33000000, "passed": true }, { "record_id": "R3168", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R3168", "field": "contact_name", "actual": "Morgan Alder6", "original": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "record_id": "R3168", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R3168", "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "original": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true }, { "record_id": "R4787", "field": "entity_id", "actual": "ALDER-SPO-19257", "original": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "record_id": "R4787", "field": "firm_name", "actual": "Alder Square 5 Capital", "original": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "record_id": "R4787", "field": "transaction_size_usd", "actual": "31200000", "original": "31200000", "expected": 31200000, "passed": true }, { "record_id": "R4787", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R4787", "field": "contact_name", "actual": "Alex Alder5", "original": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "record_id": "R4787", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4787", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-alder/turn-01/tool-01.json" }, { "record_id": "R4752", "field": "entity_id", "actual": "ALDER-SPO-13273", "original": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "record_id": "R4752", "field": "firm_name", "actual": "Alder Grove 4 Capital", "original": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "record_id": "R4752", "field": "transaction_size_usd", "actual": "29400000", "original": "29400000", "expected": 29400000, "passed": true }, { "record_id": "R4752", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R4752", "field": "contact_name", "actual": "Taylor Alder4", "original": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "record_id": "R4752", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R4752", "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "original": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true }, { "record_id": "R2273", "field": "entity_id", "actual": "ALDER-SPO-21979", "original": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "record_id": "R2273", "field": "firm_name", "actual": "Alder Park 3 Capital", "original": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "record_id": "R2273", "field": "transaction_size_usd", "actual": "27600000", "original": "27600000", "expected": 27600000, "passed": true }, { "record_id": "R2273", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2273", "field": "contact_name", "actual": "Jordan Alder3", "original": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "record_id": "R2273", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2273", "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "original": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true }, { "record_id": "R1117", "field": "entity_id", "actual": "ALDER-SPO-82903", "original": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "record_id": "R1117", "field": "firm_name", "actual": "Alder Park 2 Capital", "original": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "record_id": "R1117", "field": "transaction_size_usd", "actual": "25800000", "original": "25800000", "expected": 25800000, "passed": true }, { "record_id": "R1117", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R1117", "field": "contact_name", "actual": "Drew Alder", "original": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "record_id": "R1117", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1117", "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "original": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true }, { "record_id": "R7805", "field": "entity_id", "actual": "ALDER-SPO-80621", "original": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "record_id": "R7805", "field": "firm_name", "actual": "Alder Grove 1 Capital", "original": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "record_id": "R7805", "field": "transaction_size_usd", "actual": "24000000", "original": "24000000", "expected": 24000000, "passed": true }, { "record_id": "R7805", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R7805", "field": "contact_name", "actual": "Alex Alder1", "original": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "record_id": "R7805", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R7805", "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "original": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R3168", "passed": true, "expected": [ "S-R3168" ], "actual": [ "S-R3168" ] }, { "record_id": "R4787", "passed": true, "expected": [ "S-R4787" ], "actual": [ "S-R4787" ] }, { "record_id": "R4752", "passed": true, "expected": [ "S-R4752" ], "actual": [ "S-R4752", "U-R4752-3" ] }, { "record_id": "R2273", "passed": true, "expected": [ "S-R2273" ], "actual": [ "S-R2273", "U-R2273-2" ] }, { "record_id": "R1117", "passed": true, "expected": [ "S-R1117", "U-R1117-1" ], "actual": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R7805", "passed": true, "expected": [ "S-R5723", "S-R7756", "S-R7805" ], "actual": [ "S-R5723", "S-R7756", "S-R7805" ] } ], "source_sha256": "fed84c21a6038dd9f19f674ceb897bc58aca6ee910f5a3d106a509f6caf9096d" } }, { "case_id": "sponsor-leads-alder", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:22:55.621296+00:00", "completed_at": "2026-09-07T02:25:11.761105+00:00", "elapsed_seconds": 136.139, "cost_usd": 0.11851, "known_partial_cost_usd": 0.11851, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5464, "completion_tokens": 11168, "total_tokens": 16632, "cost": 0.11851, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 8256, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11851, "gateway_cost": 0.11851 }, "finish_reason": "stop", "elapsed_seconds": 136.119, "completed_at": "2026-09-07T02:25:11.758367+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sponsor-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-alder", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R9772", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R9772", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-86867", "expected": "ALDER-SPO-86867", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 18 Capital", "expected": "Alder Grove 18 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder18", "expected": "Morgan Alder18", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person18@sponsor-leads-alder.example", "expected": "person18@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9772" ], "reported_sources": [ "S-R9772" ] }, { "record_id": "R3168", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "field": "firm_name", "actual": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 33000000, "expected": 33000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3168" ], "reported_sources": [ "S-R3168" ] }, { "record_id": "R4787", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "field": "firm_name", "actual": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 31200000, "expected": 31200000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4787" ], "reported_sources": [ "S-R4787" ] }, { "record_id": "R4752", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4752" ], "reported_sources": [ "S-R4752" ] }, { "record_id": "R2273", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "field": "firm_name", "actual": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 27600000, "expected": 27600000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2273" ], "reported_sources": [ "S-R2273" ] }, { "record_id": "R1117", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "field": "firm_name", "actual": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 25800000, "expected": 25800000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1117", "U-R1117-1" ], "reported_sources": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R7756", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34800000, "expected": 34800000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7756", "S-R7805" ], "reported_sources": [ "S-R7756", "S-R7805" ] } ], "decision_checks": [ { "record_id": "R9772", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4022", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1654", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5723", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1117", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4787", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3335", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R1421", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2273", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1078", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R7756", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2889", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4752", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4993", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R3168", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9496", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2033", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7805", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4644", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-alder", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:23:53.473236+00:00", "completed_at": "2026-09-07T02:25:02.228726+00:00", "elapsed_seconds": 68.755, "cost_usd": 0.214, "known_partial_cost_usd": 0.214, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 7950, "completion_tokens": 6970, "total_tokens": 14920, "cost": 0.214, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1731, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.214, "gateway_cost": 0.214 }, "finish_reason": "stop", "elapsed_seconds": 68.738, "completed_at": "2026-09-07T02:25:02.226202+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sponsor-leads-alder", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-alder", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R3168", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-67037", "expected": "ALDER-SPO-67037", "passed": true }, { "field": "firm_name", "actual": "Alder Square 6 Capital", "expected": "Alder Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 33000000, "expected": 33000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Alder6", "expected": "Morgan Alder6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-alder.example", "expected": "person6@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3168" ], "reported_sources": [ "S-R3168" ] }, { "record_id": "R4787", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-19257", "expected": "ALDER-SPO-19257", "passed": true }, { "field": "firm_name", "actual": "Alder Square 5 Capital", "expected": "Alder Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 31200000, "expected": 31200000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Alder5", "expected": "Alex Alder5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4787" ], "reported_sources": [ "S-R4787" ] }, { "record_id": "R4752", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-13273", "expected": "ALDER-SPO-13273", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 4 Capital", "expected": "Alder Grove 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 29400000, "expected": 29400000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Alder4", "expected": "Taylor Alder4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-alder.example", "expected": "person4@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4752" ], "reported_sources": [ "S-R4752" ] }, { "record_id": "R2273", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-21979", "expected": "ALDER-SPO-21979", "passed": true }, { "field": "firm_name", "actual": "Alder Park 3 Capital", "expected": "Alder Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 27600000, "expected": 27600000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Alder3", "expected": "Jordan Alder3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-alder.example", "expected": "person3@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2273" ], "reported_sources": [ "S-R2273" ] }, { "record_id": "R1117", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-82903", "expected": "ALDER-SPO-82903", "passed": true }, { "field": "firm_name", "actual": "Alder Park 2 Capital", "expected": "Alder Park 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 25800000, "expected": 25800000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Alder", "expected": "Drew Alder", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-alder.example", "expected": "new.cfo@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1117", "U-R1117-1" ], "reported_sources": [ "S-R1117", "U-R1117-1" ] }, { "record_id": "R7805", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ALDER-SPO-80621", "expected": "ALDER-SPO-80621", "passed": true }, { "field": "firm_name", "actual": "Alder Grove 1 Capital", "expected": "Alder Grove 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 24000000, "expected": 24000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Alder1", "expected": "Alex Alder1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-alder.example", "expected": "person1@sponsor-leads-alder.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7805" ], "reported_sources": [ "S-R7805", "S-R5723", "S-R7756" ] } ], "decision_checks": [ { "record_id": "R9772", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4022", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1654", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R5723", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1117", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4787", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3335", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R1421", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2273", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1078", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R7756", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7814", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2889", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4752", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4993", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R3168", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9496", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R2033", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7805", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4644", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-birch", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:25:15.101Z", "observed_completed_at": "2026-09-07T02:32:52.455Z", "packet_sha256": "82b09d36ca81917e278aed00e0bd0f8f8db701855ba251b1de6703798ddde839", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/birch_phoenix_retail_prospects.xlsx", "sha256": "83ac4161ad1cdf06eedfbb31e8d0db73623225194485c221110b98d589ec6217", "bytes": 8164 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.39813775, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 94.99, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "be30ae05-633f-4cdf-81b6-1f5436bdc90e", "session_id": "545f400e-d22f-4f69-89e7-e47183497e2e", "started_at": "2026-09-06T22:26:15.504000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.25514975, 0.142988 ], "total_cost_usd": 0.39813775, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 94.99, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sponsor-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-birch", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2245", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "field": "firm_name", "actual": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13200000, "expected": 13200000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2245" ], "reported_sources": [ "S-R2245" ] }, { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "field": "firm_name", "actual": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 12480000, "expected": 12480000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968" ] }, { "record_id": "R8710", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "field": "firm_name", "actual": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11760000, "expected": 11760000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8710" ], "reported_sources": [ "S-R8710" ] }, { "record_id": "R1638", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "field": "firm_name", "actual": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11040000, "expected": 11040000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1638" ], "reported_sources": [ "S-R1638" ] }, { "record_id": "R7815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "field": "firm_name", "actual": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10320000, "expected": 10320000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7815", "U-R7815-1" ], "reported_sources": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R2431", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "field": "firm_name", "actual": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9600000, "expected": 9600000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2431" ], "reported_sources": [ "S-R2431", "S-R1077", "S-R2005" ] } ], "decision_checks": [ { "record_id": "R6206", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R1638", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2245", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6226", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2431", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3377", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1919", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6320", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9716", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7842", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8710", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6119", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4272", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1045", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1077", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8692", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R8208", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": false }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/birch_phoenix_retail_prospects.xlsx", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "Requested .csv; delivered .xlsx", "R6968 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "Prospects", "Candidate Decisions" ], "table_sheet": "Prospects", "original_record_order": [ "R2245", "R6968", "R8710", "R1638", "R7815", "R2431" ], "answer_record_order": [ "R2245", "R6968", "R8710", "R1638", "R7815", "R2431" ], "field_checks": [ { "record_id": "R2245", "field": "entity_id", "actual": "BIRCH-SPO-58038", "original": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "record_id": "R2245", "field": "firm_name", "actual": "Birch Point 6 Capital", "original": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "record_id": "R2245", "field": "transaction_size_usd", "actual": 13200000, "original": 13200000, "expected": 13200000, "passed": true }, { "record_id": "R2245", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R2245", "field": "contact_name", "actual": "Morgan Birch6", "original": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "record_id": "R2245", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2245", "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "original": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true }, { "record_id": "R6968", "field": "entity_id", "actual": "BIRCH-SPO-48685", "original": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "record_id": "R6968", "field": "firm_name", "actual": "Birch Park 5 Capital", "original": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "record_id": "R6968", "field": "transaction_size_usd", "actual": 12480000, "original": 12480000, "expected": 12480000, "passed": true }, { "record_id": "R6968", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6968", "field": "contact_name", "actual": "Alex Birch5", "original": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "record_id": "R6968", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R6968", "field": "contact_email", "actual": null, "original": null, "expected": null, "passed": false }, { "record_id": "R8710", "field": "entity_id", "actual": "BIRCH-SPO-35519", "original": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "record_id": "R8710", "field": "firm_name", "actual": "Birch Square 4 Capital", "original": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "record_id": "R8710", "field": "transaction_size_usd", "actual": 11760000, "original": 11760000, "expected": 11760000, "passed": true }, { "record_id": "R8710", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R8710", "field": "contact_name", "actual": "Taylor Birch4", "original": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "record_id": "R8710", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R8710", "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "original": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true }, { "record_id": "R1638", "field": "entity_id", "actual": "BIRCH-SPO-67736", "original": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "record_id": "R1638", "field": "firm_name", "actual": "Birch Park 3 Capital", "original": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "record_id": "R1638", "field": "transaction_size_usd", "actual": 11040000, "original": 11040000, "expected": 11040000, "passed": true }, { "record_id": "R1638", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1638", "field": "contact_name", "actual": "Jordan Birch3", "original": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "record_id": "R1638", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1638", "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "original": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true }, { "record_id": "R7815", "field": "entity_id", "actual": "BIRCH-SPO-66254", "original": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "record_id": "R7815", "field": "firm_name", "actual": "Birch Square 2 Capital", "original": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "record_id": "R7815", "field": "transaction_size_usd", "actual": 10320000, "original": 10320000, "expected": 10320000, "passed": true }, { "record_id": "R7815", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R7815", "field": "contact_name", "actual": "Drew Birch", "original": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "record_id": "R7815", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R7815", "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "original": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true }, { "record_id": "R2431", "field": "entity_id", "actual": "BIRCH-SPO-59032", "original": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "record_id": "R2431", "field": "firm_name", "actual": "Birch Square 1 Capital", "original": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "record_id": "R2431", "field": "transaction_size_usd", "actual": 9600000, "original": 9600000, "expected": 9600000, "passed": true }, { "record_id": "R2431", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R2431", "field": "contact_name", "actual": "Alex Birch1", "original": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "record_id": "R2431", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2431", "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "original": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2245", "passed": true, "expected": [ "S-R2245" ], "actual": [ "S-R2245" ] }, { "record_id": "R6968", "passed": true, "expected": [ "S-R6968" ], "actual": [ "S-R6968" ] }, { "record_id": "R8710", "passed": true, "expected": [ "S-R8710" ], "actual": [ "S-R8710", "U-R8710-3" ] }, { "record_id": "R1638", "passed": true, "expected": [ "S-R1638" ], "actual": [ "S-R1638", "U-R1638-2" ] }, { "record_id": "R7815", "passed": true, "expected": [ "S-R7815", "U-R7815-1" ], "actual": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R2431", "passed": true, "expected": [ "S-R1077", "S-R2005", "S-R2431" ], "actual": [ "S-R1077", "S-R2005", "S-R2431" ] } ], "source_sha256": "83ac4161ad1cdf06eedfbb31e8d0db73623225194485c221110b98d589ec6217" } }, { "case_id": "sponsor-leads-birch", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:25:02.229864+00:00", "completed_at": "2026-09-07T02:29:30.894289+00:00", "elapsed_seconds": 268.662, "cost_usd": 0.25125275, "known_partial_cost_usd": 0.25125275, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 3, "tool_calls": 3, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6216, "completion_tokens": 8483, "total_tokens": 14699, "cost": 0.0926, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 8064, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0926, "gateway_cost": 0.0926 }, "finish_reason": "tool_calls", "elapsed_seconds": 87.668, "completed_at": "2026-09-07T02:26:29.914450+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6629, "completion_tokens": 6847, "total_tokens": 13476, "cost": 0.06984425, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6144, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6400, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06984425, "gateway_cost": 0.06984425 }, "finish_reason": "tool_calls", "elapsed_seconds": 71.983, "completed_at": "2026-09-07T02:27:41.903826+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7042, "completion_tokens": 8735, "total_tokens": 15777, "cost": 0.0888085, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6528, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6720, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0888085, "gateway_cost": 0.0888085 }, "finish_reason": "tool_calls", "elapsed_seconds": 108.984, "completed_at": "2026-09-07T02:29:30.891464+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "30bb9b47ea8dbf7fbf1375d6a75866fc20ba3d76e75ad198bc79c76fd725032c", "bytes": 887 }, { "path": "artifacts/prospects-v2.csv", "sha256": "e83781f557e6a26967ede0c0651c6c1e9280bfd5f7db0edaee26624b7e5a8f20", "bytes": 887 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-birch", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2245", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "field": "firm_name", "actual": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13200000, "expected": 13200000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2245" ], "reported_sources": [ "S-R2245" ] }, { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "field": "firm_name", "actual": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 12480000, "expected": 12480000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968" ] }, { "record_id": "R8710", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "field": "firm_name", "actual": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11760000, "expected": 11760000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8710" ], "reported_sources": [ "S-R8710" ] }, { "record_id": "R1638", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "field": "firm_name", "actual": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11040000, "expected": 11040000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1638" ], "reported_sources": [ "S-R1638" ] }, { "record_id": "R7815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "field": "firm_name", "actual": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10320000, "expected": 10320000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7815", "U-R7815-1" ], "reported_sources": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R2005", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "field": "firm_name", "actual": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13920000, "expected": 13920000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1077", "S-R2005" ], "reported_sources": [ "S-R2005", "S-R1077" ] } ], "decision_checks": [ { "record_id": "R6206", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R1638", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2245", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6226", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2431", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3377", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1919", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6320", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9716", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7842", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8710", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6119", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4272", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1045", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1077", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8692", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R8208", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v2.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R6968 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2245", "R6968", "R8710", "R1638", "R7815", "R2005" ], "answer_record_order": [ "R2245", "R6968", "R8710", "R1638", "R7815", "R2005" ], "field_checks": [ { "record_id": "R2245", "field": "entity_id", "actual": "BIRCH-SPO-58038", "original": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "record_id": "R2245", "field": "firm_name", "actual": "Birch Point 6 Capital", "original": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "record_id": "R2245", "field": "transaction_size_usd", "actual": "13200000", "original": "13200000", "expected": 13200000, "passed": true }, { "record_id": "R2245", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R2245", "field": "contact_name", "actual": "Morgan Birch6", "original": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "record_id": "R2245", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2245", "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "original": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true }, { "record_id": "R6968", "field": "entity_id", "actual": "BIRCH-SPO-48685", "original": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "record_id": "R6968", "field": "firm_name", "actual": "Birch Park 5 Capital", "original": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "record_id": "R6968", "field": "transaction_size_usd", "actual": "12480000", "original": "12480000", "expected": 12480000, "passed": true }, { "record_id": "R6968", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6968", "field": "contact_name", "actual": "Alex Birch5", "original": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "record_id": "R6968", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R6968", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-birch/turn-02/tool-01.json" }, { "record_id": "R8710", "field": "entity_id", "actual": "BIRCH-SPO-35519", "original": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "record_id": "R8710", "field": "firm_name", "actual": "Birch Square 4 Capital", "original": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "record_id": "R8710", "field": "transaction_size_usd", "actual": "11760000", "original": "11760000", "expected": 11760000, "passed": true }, { "record_id": "R8710", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R8710", "field": "contact_name", "actual": "Taylor Birch4", "original": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "record_id": "R8710", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R8710", "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "original": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true }, { "record_id": "R1638", "field": "entity_id", "actual": "BIRCH-SPO-67736", "original": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "record_id": "R1638", "field": "firm_name", "actual": "Birch Park 3 Capital", "original": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "record_id": "R1638", "field": "transaction_size_usd", "actual": "11040000", "original": "11040000", "expected": 11040000, "passed": true }, { "record_id": "R1638", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1638", "field": "contact_name", "actual": "Jordan Birch3", "original": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "record_id": "R1638", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1638", "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "original": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true }, { "record_id": "R7815", "field": "entity_id", "actual": "BIRCH-SPO-66254", "original": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "record_id": "R7815", "field": "firm_name", "actual": "Birch Square 2 Capital", "original": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "record_id": "R7815", "field": "transaction_size_usd", "actual": "10320000", "original": "10320000", "expected": 10320000, "passed": true }, { "record_id": "R7815", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R7815", "field": "contact_name", "actual": "Drew Birch", "original": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "record_id": "R7815", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R7815", "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "original": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true }, { "record_id": "R2005", "field": "entity_id", "actual": "BIRCH-SPO-59032", "original": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "record_id": "R2005", "field": "firm_name", "actual": "Birch Square 1 Capital", "original": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "record_id": "R2005", "field": "transaction_size_usd", "actual": "13920000", "original": "13920000", "expected": 13920000, "passed": true }, { "record_id": "R2005", "field": "event_date", "actual": "2026-02-11", "original": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "record_id": "R2005", "field": "contact_name", "actual": "Alex Birch1", "original": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "record_id": "R2005", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2005", "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "original": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R2245", "passed": true, "expected": [ "S-R2245" ], "actual": [ "S-R2245" ] }, { "record_id": "R6968", "passed": true, "expected": [ "S-R6968" ], "actual": [ "S-R6968" ] }, { "record_id": "R8710", "passed": true, "expected": [ "S-R8710" ], "actual": [ "S-R8710" ] }, { "record_id": "R1638", "passed": true, "expected": [ "S-R1638" ], "actual": [ "S-R1638" ] }, { "record_id": "R7815", "passed": true, "expected": [ "S-R7815", "U-R7815-1" ], "actual": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R2005", "passed": true, "expected": [ "S-R1077", "S-R2005" ], "actual": [ "S-R1077", "S-R2005" ] } ], "source_sha256": "e83781f557e6a26967ede0c0651c6c1e9280bfd5f7db0edaee26624b7e5a8f20" } }, { "case_id": "sponsor-leads-birch", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T02:25:11.761720+00:00", "completed_at": "2026-09-07T02:26:46.289258+00:00", "elapsed_seconds": 94.527, "cost_usd": 0.39698, "known_partial_cost_usd": 0.39698, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 3, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10221, "completion_tokens": 2218, "total_tokens": 12439, "cost": 0.106555, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1343, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.106555, "gateway_cost": 0.106555 }, "finish_reason": "tool_calls", "elapsed_seconds": 24.07, "completed_at": "2026-09-07T02:25:35.844354+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 11125, "completion_tokens": 1453, "total_tokens": 12578, "cost": 0.09195, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 577, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.09195, "gateway_cost": 0.09195 }, "finish_reason": "tool_calls", "elapsed_seconds": 15.978, "completed_at": "2026-09-07T02:25:51.824304+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 12030, "completion_tokens": 5533, "total_tokens": 17563, "cost": 0.198475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1034, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.198475, "gateway_cost": 0.198475 }, "finish_reason": "stop", "elapsed_seconds": 54.456, "completed_at": "2026-09-07T02:26:46.286708+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "084c30e25f71b2ee8b49ade4e677a020c401d3ee92e765fc55de8da3d0604836", "bytes": 1256 }, { "path": "artifacts/prospects-v2.csv", "sha256": "8b7cd9de5da8746989f7ece2f283e1ff252a4f6beed4371ce62165309ef7677d", "bytes": 1267 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-birch", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2245", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "field": "firm_name", "actual": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13200000, "expected": 13200000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2245" ], "reported_sources": [ "S-R2245" ] }, { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "field": "firm_name", "actual": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 12480000, "expected": 12480000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968" ] }, { "record_id": "R8710", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "field": "firm_name", "actual": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11760000, "expected": 11760000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8710" ], "reported_sources": [ "S-R8710" ] }, { "record_id": "R1638", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "field": "firm_name", "actual": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11040000, "expected": 11040000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1638" ], "reported_sources": [ "S-R1638" ] }, { "record_id": "R7815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "field": "firm_name", "actual": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10320000, "expected": 10320000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7815", "U-R7815-1" ], "reported_sources": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R1077", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "field": "firm_name", "actual": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9600000, "expected": 9600000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1077" ], "reported_sources": [ "S-R1077" ] } ], "decision_checks": [ { "record_id": "R6206", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R1638", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2245", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6226", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2431", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3377", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1919", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6320", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9716", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7842", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8710", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6119", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4272", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1045", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1077", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8692", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R8208", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v2.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R6968 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2245", "R6968", "R8710", "R1638", "R7815", "R1077" ], "answer_record_order": [ "R2245", "R6968", "R8710", "R1638", "R7815", "R1077" ], "field_checks": [ { "record_id": "R2245", "field": "entity_id", "actual": "BIRCH-SPO-58038", "original": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "record_id": "R2245", "field": "firm_name", "actual": "Birch Point 6 Capital", "original": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "record_id": "R2245", "field": "transaction_size_usd", "actual": "13200000", "original": "13200000", "expected": 13200000, "passed": true }, { "record_id": "R2245", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R2245", "field": "contact_name", "actual": "Morgan Birch6", "original": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "record_id": "R2245", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2245", "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "original": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true }, { "record_id": "R6968", "field": "entity_id", "actual": "BIRCH-SPO-48685", "original": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "record_id": "R6968", "field": "firm_name", "actual": "Birch Park 5 Capital", "original": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "record_id": "R6968", "field": "transaction_size_usd", "actual": "12480000", "original": "12480000", "expected": 12480000, "passed": true }, { "record_id": "R6968", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R6968", "field": "contact_name", "actual": "Alex Birch5", "original": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "record_id": "R6968", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R6968", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-birch/turn-02/tool-01.json" }, { "record_id": "R8710", "field": "entity_id", "actual": "BIRCH-SPO-35519", "original": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "record_id": "R8710", "field": "firm_name", "actual": "Birch Square 4 Capital", "original": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "record_id": "R8710", "field": "transaction_size_usd", "actual": "11760000", "original": "11760000", "expected": 11760000, "passed": true }, { "record_id": "R8710", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R8710", "field": "contact_name", "actual": "Taylor Birch4", "original": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "record_id": "R8710", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R8710", "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "original": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true }, { "record_id": "R1638", "field": "entity_id", "actual": "BIRCH-SPO-67736", "original": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "record_id": "R1638", "field": "firm_name", "actual": "Birch Park 3 Capital", "original": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "record_id": "R1638", "field": "transaction_size_usd", "actual": "11040000", "original": "11040000", "expected": 11040000, "passed": true }, { "record_id": "R1638", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1638", "field": "contact_name", "actual": "Jordan Birch3", "original": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "record_id": "R1638", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1638", "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "original": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true }, { "record_id": "R7815", "field": "entity_id", "actual": "BIRCH-SPO-66254", "original": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "record_id": "R7815", "field": "firm_name", "actual": "Birch Square 2 Capital", "original": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "record_id": "R7815", "field": "transaction_size_usd", "actual": "10320000", "original": "10320000", "expected": 10320000, "passed": true }, { "record_id": "R7815", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R7815", "field": "contact_name", "actual": "Drew Birch", "original": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "record_id": "R7815", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R7815", "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "original": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true }, { "record_id": "R1077", "field": "entity_id", "actual": "BIRCH-SPO-59032", "original": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "record_id": "R1077", "field": "firm_name", "actual": "Birch Square 1 Capital", "original": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "record_id": "R1077", "field": "transaction_size_usd", "actual": "9600000", "original": "9600000", "expected": 9600000, "passed": true }, { "record_id": "R1077", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R1077", "field": "contact_name", "actual": "Alex Birch1", "original": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "record_id": "R1077", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1077", "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "original": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R2245", "passed": true, "expected": [ "S-R2245" ], "actual": [ "S-R2245" ] }, { "record_id": "R6968", "passed": true, "expected": [ "S-R6968" ], "actual": [ "S-R6968" ] }, { "record_id": "R8710", "passed": true, "expected": [ "S-R8710" ], "actual": [ "S-R8710", "U-R8710-3" ] }, { "record_id": "R1638", "passed": true, "expected": [ "S-R1638" ], "actual": [ "S-R1638", "U-R1638-2" ] }, { "record_id": "R7815", "passed": true, "expected": [ "S-R7815", "U-R7815-1" ], "actual": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R1077", "passed": true, "expected": [ "S-R1077" ], "actual": [ "S-R1077", "S-R2005", "S-R2431" ] } ], "source_sha256": "8b7cd9de5da8746989f7ece2f283e1ff252a4f6beed4371ce62165309ef7677d" } }, { "case_id": "sponsor-leads-birch", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:26:16.935171+00:00", "completed_at": "2026-09-07T02:27:52.381123+00:00", "elapsed_seconds": 95.444, "cost_usd": 0.1088775, "known_partial_cost_usd": 0.1088775, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5470, "completion_tokens": 10204, "total_tokens": 15674, "cost": 0.1088775, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7424, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.1088775, "gateway_cost": 0.1088775 }, "finish_reason": "stop", "elapsed_seconds": 95.424, "completed_at": "2026-09-07T02:27:52.379767+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sponsor-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-birch", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2245", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "field": "firm_name", "actual": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13200000, "expected": 13200000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2245" ], "reported_sources": [ "S-R2245" ] }, { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "field": "firm_name", "actual": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 12480000, "expected": 12480000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968" ] }, { "record_id": "R8710", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "field": "firm_name", "actual": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11760000, "expected": 11760000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8710" ], "reported_sources": [ "S-R8710" ] }, { "record_id": "R1638", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "field": "firm_name", "actual": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11040000, "expected": 11040000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1638" ], "reported_sources": [ "S-R1638" ] }, { "record_id": "R7815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "field": "firm_name", "actual": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10320000, "expected": 10320000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7815", "U-R7815-1" ], "reported_sources": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R2005", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "field": "firm_name", "actual": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13920000, "expected": 13920000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1077", "S-R2005" ], "reported_sources": [ "S-R2005", "S-R1077" ] } ], "decision_checks": [ { "record_id": "R6206", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R1638", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2245", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6226", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2431", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3377", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1919", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6320", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9716", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7842", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8710", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6119", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4272", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1045", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1077", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8692", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R8208", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-birch", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T02:26:46.290724+00:00", "completed_at": "2026-09-07T02:27:47.573276+00:00", "elapsed_seconds": 61.281, "cost_usd": 0.19673, "known_partial_cost_usd": 0.19673, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 7936, "completion_tokens": 6282, "total_tokens": 14218, "cost": 0.19673, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1239, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.19673, "gateway_cost": 0.19673 }, "finish_reason": "stop", "elapsed_seconds": 61.252, "completed_at": "2026-09-07T02:27:47.569103+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sponsor-leads-birch", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-birch", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2245", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-58038", "expected": "BIRCH-SPO-58038", "passed": true }, { "field": "firm_name", "actual": "Birch Point 6 Capital", "expected": "Birch Point 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 13200000, "expected": 13200000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Birch6", "expected": "Morgan Birch6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-birch.example", "expected": "person6@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2245" ], "reported_sources": [ "S-R2245" ] }, { "record_id": "R6968", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-48685", "expected": "BIRCH-SPO-48685", "passed": true }, { "field": "firm_name", "actual": "Birch Park 5 Capital", "expected": "Birch Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 12480000, "expected": 12480000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Birch5", "expected": "Alex Birch5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R6968" ], "reported_sources": [ "S-R6968" ] }, { "record_id": "R8710", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-35519", "expected": "BIRCH-SPO-35519", "passed": true }, { "field": "firm_name", "actual": "Birch Square 4 Capital", "expected": "Birch Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11760000, "expected": 11760000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Birch4", "expected": "Taylor Birch4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-birch.example", "expected": "person4@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8710" ], "reported_sources": [ "S-R8710" ] }, { "record_id": "R1638", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-67736", "expected": "BIRCH-SPO-67736", "passed": true }, { "field": "firm_name", "actual": "Birch Park 3 Capital", "expected": "Birch Park 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11040000, "expected": 11040000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Birch3", "expected": "Jordan Birch3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-birch.example", "expected": "person3@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1638" ], "reported_sources": [ "S-R1638" ] }, { "record_id": "R7815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-66254", "expected": "BIRCH-SPO-66254", "passed": true }, { "field": "firm_name", "actual": "Birch Square 2 Capital", "expected": "Birch Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10320000, "expected": 10320000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Birch", "expected": "Drew Birch", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-birch.example", "expected": "new.cfo@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R7815", "U-R7815-1" ], "reported_sources": [ "S-R7815", "U-R7815-1" ] }, { "record_id": "R1077", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "BIRCH-SPO-59032", "expected": "BIRCH-SPO-59032", "passed": true }, { "field": "firm_name", "actual": "Birch Square 1 Capital", "expected": "Birch Square 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9600000, "expected": 9600000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Birch1", "expected": "Alex Birch1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-birch.example", "expected": "person1@sponsor-leads-birch.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1077" ], "reported_sources": [ "S-R1077", "S-R2431", "S-R2005" ] } ], "decision_checks": [ { "record_id": "R6206", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R1638", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2245", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6226", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R2431", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2005", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3377", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1919", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6320", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9716", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R7842", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R7815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8710", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6119", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4272", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R1045", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R6968", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1077", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R8692", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R8208", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-cypress", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:32:49.338Z", "observed_completed_at": "2026-09-07T02:36:26.467Z", "packet_sha256": "7b982f2c0bc1bf8dfa65eada5a0e28255053bc7374bf62a2fd898da9aec5899d", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/sponsor_leads_cypress_columbus_industrial.csv", "sha256": "a9a39b67e3180a5307c903b82931426c902d7cb2f93f3b818b194a0f0127a33d", "bytes": 1113 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.33675425000000003, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 78.96, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "678fa6c6-5828-4d4e-a5de-d0752672793c", "session_id": "b96de691-6507-45c6-8aa2-dee042a6177f", "started_at": "2026-09-06T22:33:31.932000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.199606, 0.13714825 ], "total_cost_usd": 0.33675425000000003, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 78.96, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sponsor-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-cypress", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 49, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R1826", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] } ], "selected_checks": [ { "record_id": "R1826", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-94665", "expected": "CYPRESS-SPO-94665", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 18 Capital", "expected": "Cypress Crossing 18 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9800000, "expected": 9800000, "passed": true }, { "field": "event_date", "actual": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "field": "contact_name", "actual": "Morgan Cypress18", "expected": "Morgan Cypress18", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person18@sponsor-leads-cypress.example", "expected": "person18@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1826" ], "reported_sources": [ "S-R1826" ] }, { "record_id": "R9249", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11000000, "expected": 11000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9249" ], "reported_sources": [ "S-R9249" ] }, { "record_id": "R3919", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "field": "firm_name", "actual": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10400000, "expected": 10400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3919" ], "reported_sources": [ "S-R3919" ] }, { "record_id": "R2909", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9800000, "expected": 9800000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2909" ], "reported_sources": [ "S-R2909" ] }, { "record_id": "R4338", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9200000, "expected": 9200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4338" ], "reported_sources": [ "S-R4338" ] }, { "record_id": "R9956", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "field": "firm_name", "actual": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8600000, "expected": 8600000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9956", "U-R9956-1" ], "reported_sources": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R4738", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8000000, "expected": 8000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4738" ], "reported_sources": [ "S-R4738", "S-R9815" ] } ], "decision_checks": [ { "record_id": "R9956", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3919", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5272", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R7976", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2395", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R4766", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8528", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4857", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3688", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R9249", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2909", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9289", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4738", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6278", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6741", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R9815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2141", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7602", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R4338", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1826", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/sponsor_leads_cypress_columbus_industrial.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "R3919 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R1826", "R9249", "R3919", "R2909", "R4338", "R9956", "R4738" ], "answer_record_order": [ "R1826", "R9249", "R3919", "R2909", "R4338", "R9956", "R4738" ], "field_checks": [ { "record_id": "R1826", "field": "entity_id", "actual": "CYPRESS-SPO-94665", "original": "CYPRESS-SPO-94665", "expected": "CYPRESS-SPO-94665", "passed": true }, { "record_id": "R1826", "field": "firm_name", "actual": "Cypress Crossing 18 Capital", "original": "Cypress Crossing 18 Capital", "expected": "Cypress Crossing 18 Capital", "passed": true }, { "record_id": "R1826", "field": "transaction_size_usd", "actual": "9800000", "original": "9800000", "expected": 9800000, "passed": true }, { "record_id": "R1826", "field": "event_date", "actual": "2026-07-22", "original": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "record_id": "R1826", "field": "contact_name", "actual": "Morgan Cypress18", "original": "Morgan Cypress18", "expected": "Morgan Cypress18", "passed": true }, { "record_id": "R1826", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R1826", "field": "contact_email", "actual": "person18@sponsor-leads-cypress.example", "original": "person18@sponsor-leads-cypress.example", "expected": "person18@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R9249", "field": "entity_id", "actual": "CYPRESS-SPO-30009", "original": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "record_id": "R9249", "field": "firm_name", "actual": "Cypress Square 6 Capital", "original": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "record_id": "R9249", "field": "transaction_size_usd", "actual": "11000000", "original": "11000000", "expected": 11000000, "passed": true }, { "record_id": "R9249", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R9249", "field": "contact_name", "actual": "Morgan Cypress6", "original": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "record_id": "R9249", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R9249", "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "original": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R3919", "field": "entity_id", "actual": "CYPRESS-SPO-21730", "original": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "record_id": "R3919", "field": "firm_name", "actual": "Cypress Park 5 Capital", "original": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "record_id": "R3919", "field": "transaction_size_usd", "actual": "10400000", "original": "10400000", "expected": 10400000, "passed": true }, { "record_id": "R3919", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R3919", "field": "contact_name", "actual": "Alex Cypress5", "original": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "record_id": "R3919", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R3919", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R2909", "field": "entity_id", "actual": "CYPRESS-SPO-81408", "original": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "record_id": "R2909", "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "original": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "record_id": "R2909", "field": "transaction_size_usd", "actual": "9800000", "original": "9800000", "expected": 9800000, "passed": true }, { "record_id": "R2909", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R2909", "field": "contact_name", "actual": "Taylor Cypress4", "original": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "record_id": "R2909", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2909", "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "original": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R4338", "field": "entity_id", "actual": "CYPRESS-SPO-51531", "original": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "record_id": "R4338", "field": "firm_name", "actual": "Cypress Square 3 Capital", "original": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "record_id": "R4338", "field": "transaction_size_usd", "actual": "9200000", "original": "9200000", "expected": 9200000, "passed": true }, { "record_id": "R4338", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R4338", "field": "contact_name", "actual": "Jordan Cypress3", "original": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "record_id": "R4338", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4338", "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "original": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R9956", "field": "entity_id", "actual": "CYPRESS-SPO-38583", "original": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "record_id": "R9956", "field": "firm_name", "actual": "Cypress Point 2 Capital", "original": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "record_id": "R9956", "field": "transaction_size_usd", "actual": "8600000", "original": "8600000", "expected": 8600000, "passed": true }, { "record_id": "R9956", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9956", "field": "contact_name", "actual": "Drew Cypress", "original": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "record_id": "R9956", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9956", "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "original": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R4738", "field": "entity_id", "actual": "CYPRESS-SPO-90059", "original": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "record_id": "R4738", "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "original": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "record_id": "R4738", "field": "transaction_size_usd", "actual": "8000000", "original": "8000000", "expected": 8000000, "passed": true }, { "record_id": "R4738", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R4738", "field": "contact_name", "actual": "Alex Cypress1", "original": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "record_id": "R4738", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4738", "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "original": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R1826", "passed": true, "expected": [ "S-R1826" ], "actual": [ "S-R1826" ] }, { "record_id": "R9249", "passed": true, "expected": [ "S-R9249" ], "actual": [ "S-R9249" ] }, { "record_id": "R3919", "passed": true, "expected": [ "S-R3919" ], "actual": [ "S-R3919" ] }, { "record_id": "R2909", "passed": true, "expected": [ "S-R2909" ], "actual": [ "S-R2909" ] }, { "record_id": "R4338", "passed": true, "expected": [ "S-R4338" ], "actual": [ "S-R4338" ] }, { "record_id": "R9956", "passed": true, "expected": [ "S-R9956", "U-R9956-1" ], "actual": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R4738", "passed": true, "expected": [ "S-R4738", "S-R9815" ], "actual": [ "S-R4738", "S-R9815" ] } ], "source_sha256": "a9a39b67e3180a5307c903b82931426c902d7cb2f93f3b818b194a0f0127a33d" } }, { "case_id": "sponsor-leads-cypress", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:22:40.137992+00:00", "completed_at": "2026-09-07T03:27:50.875824+00:00", "elapsed_seconds": 310.737, "cost_usd": 0.34930875, "known_partial_cost_usd": 0.34930875, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 5, "tool_calls": 5, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6214, "completion_tokens": 8066, "total_tokens": 14280, "cost": 0.0884275, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7680, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0884275, "gateway_cost": 0.0884275 }, "finish_reason": "tool_calls", "elapsed_seconds": 75.848, "completed_at": "2026-09-07T03:23:56.003657+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6622, "completion_tokens": 977, "total_tokens": 7599, "cost": 0.0111355, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6144, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 0, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0111355, "gateway_cost": 0.0111355 }, "finish_reason": "tool_calls", "elapsed_seconds": 6.691, "completed_at": "2026-09-07T03:24:02.697276+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7648, "completion_tokens": 6517, "total_tokens": 14165, "cost": 0.067386, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6528, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6080, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.067386, "gateway_cost": 0.067386 }, "finish_reason": "tool_calls", "elapsed_seconds": 69.303, "completed_at": "2026-09-07T03:25:12.510417+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 8056, "completion_tokens": 7486, "total_tokens": 15542, "cost": 0.076434, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7552, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6464, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.076434, "gateway_cost": 0.076434 }, "finish_reason": "tool_calls", "elapsed_seconds": 67.104, "completed_at": "2026-09-07T03:26:19.618500+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 9075, "completion_tokens": 10351, "total_tokens": 19426, "cost": 0.10592575, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 7936, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 8064, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.10592575, "gateway_cost": 0.10592575 }, "finish_reason": "tool_calls", "elapsed_seconds": 90.753, "completed_at": "2026-09-07T03:27:50.869368+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "f9ac3d474894a46e0384021a8edfdd62464d4302b96655511df0e5dba7bc6817", "bytes": 936 }, { "path": "artifacts/prospects-v2.csv", "sha256": "f9ac3d474894a46e0384021a8edfdd62464d4302b96655511df0e5dba7bc6817", "bytes": 936 }, { "path": "artifacts/underwriting-v1.xlsx", "sha256": "a542080820f6dd00472399d568b558a124852edeba719bfcc8c4ebbbfcabe6e6", "bytes": 4344 }, { "path": "artifacts/underwriting-v2.xlsx", "sha256": "99052c820fcd54ecc2e6bcbe34604ac26228f6147fd53b3df219a35e56236c7d", "bytes": 4337 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-cypress", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R9249", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11000000, "expected": 11000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9249" ], "reported_sources": [ "S-R9249" ] }, { "record_id": "R3919", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "field": "firm_name", "actual": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10400000, "expected": 10400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3919" ], "reported_sources": [ "S-R3919" ] }, { "record_id": "R2909", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9800000, "expected": 9800000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2909" ], "reported_sources": [ "S-R2909" ] }, { "record_id": "R4338", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9200000, "expected": 9200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4338" ], "reported_sources": [ "S-R4338" ] }, { "record_id": "R9956", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "field": "firm_name", "actual": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8600000, "expected": 8600000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9956", "U-R9956-1" ], "reported_sources": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R2141", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11600000, "expected": 11600000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2141", "S-R9815" ], "reported_sources": [ "S-R2141", "S-R9815" ] } ], "decision_checks": [ { "record_id": "R9956", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3919", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5272", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R7976", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2395", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R4766", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8528", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4857", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3688", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R9249", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2909", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9289", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4738", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6278", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6741", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R9815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2141", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7602", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R4338", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1826", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v2.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R3919 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R9249", "R3919", "R2909", "R4338", "R9956", "R2141" ], "answer_record_order": [ "R9249", "R3919", "R2909", "R4338", "R9956", "R2141" ], "field_checks": [ { "record_id": "R9249", "field": "entity_id", "actual": "CYPRESS-SPO-30009", "original": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "record_id": "R9249", "field": "firm_name", "actual": "Cypress Square 6 Capital", "original": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "record_id": "R9249", "field": "transaction_size_usd", "actual": "11000000", "original": "11000000", "expected": 11000000, "passed": true }, { "record_id": "R9249", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R9249", "field": "contact_name", "actual": "Morgan Cypress6", "original": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "record_id": "R9249", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R9249", "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "original": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R3919", "field": "entity_id", "actual": "CYPRESS-SPO-21730", "original": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "record_id": "R3919", "field": "firm_name", "actual": "Cypress Park 5 Capital", "original": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "record_id": "R3919", "field": "transaction_size_usd", "actual": "10400000", "original": "10400000", "expected": 10400000, "passed": true }, { "record_id": "R3919", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R3919", "field": "contact_name", "actual": "Alex Cypress5", "original": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "record_id": "R3919", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R3919", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-cypress/turn-03/tool-01.json" }, { "record_id": "R2909", "field": "entity_id", "actual": "CYPRESS-SPO-81408", "original": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "record_id": "R2909", "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "original": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "record_id": "R2909", "field": "transaction_size_usd", "actual": "9800000", "original": "9800000", "expected": 9800000, "passed": true }, { "record_id": "R2909", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R2909", "field": "contact_name", "actual": "Taylor Cypress4", "original": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "record_id": "R2909", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2909", "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "original": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R4338", "field": "entity_id", "actual": "CYPRESS-SPO-51531", "original": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "record_id": "R4338", "field": "firm_name", "actual": "Cypress Square 3 Capital", "original": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "record_id": "R4338", "field": "transaction_size_usd", "actual": "9200000", "original": "9200000", "expected": 9200000, "passed": true }, { "record_id": "R4338", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R4338", "field": "contact_name", "actual": "Jordan Cypress3", "original": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "record_id": "R4338", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4338", "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "original": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R9956", "field": "entity_id", "actual": "CYPRESS-SPO-38583", "original": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "record_id": "R9956", "field": "firm_name", "actual": "Cypress Point 2 Capital", "original": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "record_id": "R9956", "field": "transaction_size_usd", "actual": "8600000", "original": "8600000", "expected": 8600000, "passed": true }, { "record_id": "R9956", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9956", "field": "contact_name", "actual": "Drew Cypress", "original": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "record_id": "R9956", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9956", "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "original": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R2141", "field": "entity_id", "actual": "CYPRESS-SPO-90059", "original": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "record_id": "R2141", "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "original": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "record_id": "R2141", "field": "transaction_size_usd", "actual": "11600000", "original": "11600000", "expected": 11600000, "passed": true }, { "record_id": "R2141", "field": "event_date", "actual": "2026-02-11", "original": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "record_id": "R2141", "field": "contact_name", "actual": "Alex Cypress1", "original": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "record_id": "R2141", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2141", "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "original": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R9249", "passed": true, "expected": [ "S-R9249" ], "actual": [ "S-R9249" ] }, { "record_id": "R3919", "passed": true, "expected": [ "S-R3919" ], "actual": [ "S-R3919" ] }, { "record_id": "R2909", "passed": true, "expected": [ "S-R2909" ], "actual": [ "S-R2909" ] }, { "record_id": "R4338", "passed": true, "expected": [ "S-R4338" ], "actual": [ "S-R4338" ] }, { "record_id": "R9956", "passed": true, "expected": [ "S-R9956", "U-R9956-1" ], "actual": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R2141", "passed": true, "expected": [ "S-R2141", "S-R9815" ], "actual": [ "S-R2141", "S-R9815" ] } ], "source_sha256": "f9ac3d474894a46e0384021a8edfdd62464d4302b96655511df0e5dba7bc6817" } }, { "case_id": "sponsor-leads-cypress", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:23:14.328455+00:00", "completed_at": "2026-09-07T03:24:24.393786+00:00", "elapsed_seconds": 70.065, "cost_usd": 0.28124499999999997, "known_partial_cost_usd": 0.28124499999999997, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 1, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10329, "completion_tokens": 2405, "total_tokens": 12734, "cost": 0.11177, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1633, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11177, "gateway_cost": 0.11177 }, "finish_reason": "tool_calls", "elapsed_seconds": 24.833, "completed_at": "2026-09-07T03:23:39.179488+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 11195, "completion_tokens": 4540, "total_tokens": 15735, "cost": 0.169475, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 415, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.169475, "gateway_cost": 0.169475 }, "finish_reason": "stop", "elapsed_seconds": 45.209, "completed_at": "2026-09-07T03:24:24.391714+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "ce67e0718f3026eb2874d60176cdc3fc2c4395cef709ababd886f12325807ed5", "bytes": 1202 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-cypress", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R9249", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11000000, "expected": 11000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9249" ], "reported_sources": [ "S-R9249" ] }, { "record_id": "R3919", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "field": "firm_name", "actual": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10400000, "expected": 10400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3919" ], "reported_sources": [ "S-R3919" ] }, { "record_id": "R2909", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9800000, "expected": 9800000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2909" ], "reported_sources": [ "S-R2909" ] }, { "record_id": "R4338", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9200000, "expected": 9200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4338" ], "reported_sources": [ "S-R4338" ] }, { "record_id": "R9956", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "field": "firm_name", "actual": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8600000, "expected": 8600000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9956", "U-R9956-1" ], "reported_sources": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R9815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8000000, "expected": 8000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9815" ], "reported_sources": [ "S-R9815", "S-R4738", "S-R2141" ] } ], "decision_checks": [ { "record_id": "R9956", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3919", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5272", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R7976", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2395", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R4766", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8528", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4857", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3688", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R9249", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2909", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9289", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4738", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6278", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6741", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R9815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2141", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7602", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R4338", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1826", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R9249", "R3919", "R2909", "R4338", "R9956", "R9815" ], "answer_record_order": [ "R9249", "R3919", "R2909", "R4338", "R9956", "R9815" ], "field_checks": [ { "record_id": "R9249", "field": "entity_id", "actual": "CYPRESS-SPO-30009", "original": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "record_id": "R9249", "field": "firm_name", "actual": "Cypress Square 6 Capital", "original": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "record_id": "R9249", "field": "transaction_size_usd", "actual": "11000000", "original": "11000000", "expected": 11000000, "passed": true }, { "record_id": "R9249", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R9249", "field": "contact_name", "actual": "Morgan Cypress6", "original": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "record_id": "R9249", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R9249", "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "original": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R3919", "field": "entity_id", "actual": "CYPRESS-SPO-21730", "original": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "record_id": "R3919", "field": "firm_name", "actual": "Cypress Park 5 Capital", "original": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "record_id": "R3919", "field": "transaction_size_usd", "actual": "10400000", "original": "10400000", "expected": 10400000, "passed": true }, { "record_id": "R3919", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R3919", "field": "contact_name", "actual": "Alex Cypress5", "original": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "record_id": "R3919", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R3919", "field": "contact_email", "actual": "null", "original": "null", "expected": null, "passed": true }, { "record_id": "R2909", "field": "entity_id", "actual": "CYPRESS-SPO-81408", "original": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "record_id": "R2909", "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "original": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "record_id": "R2909", "field": "transaction_size_usd", "actual": "9800000", "original": "9800000", "expected": 9800000, "passed": true }, { "record_id": "R2909", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R2909", "field": "contact_name", "actual": "Taylor Cypress4", "original": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "record_id": "R2909", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2909", "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "original": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R4338", "field": "entity_id", "actual": "CYPRESS-SPO-51531", "original": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "record_id": "R4338", "field": "firm_name", "actual": "Cypress Square 3 Capital", "original": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "record_id": "R4338", "field": "transaction_size_usd", "actual": "9200000", "original": "9200000", "expected": 9200000, "passed": true }, { "record_id": "R4338", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R4338", "field": "contact_name", "actual": "Jordan Cypress3", "original": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "record_id": "R4338", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R4338", "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "original": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R9956", "field": "entity_id", "actual": "CYPRESS-SPO-38583", "original": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "record_id": "R9956", "field": "firm_name", "actual": "Cypress Point 2 Capital", "original": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "record_id": "R9956", "field": "transaction_size_usd", "actual": "8600000", "original": "8600000", "expected": 8600000, "passed": true }, { "record_id": "R9956", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9956", "field": "contact_name", "actual": "Drew Cypress", "original": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "record_id": "R9956", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9956", "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "original": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true }, { "record_id": "R9815", "field": "entity_id", "actual": "CYPRESS-SPO-90059", "original": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "record_id": "R9815", "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "original": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "record_id": "R9815", "field": "transaction_size_usd", "actual": "8000000", "original": "8000000", "expected": 8000000, "passed": true }, { "record_id": "R9815", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R9815", "field": "contact_name", "actual": "Alex Cypress1", "original": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "record_id": "R9815", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9815", "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "original": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R9249", "passed": true, "expected": [ "S-R9249" ], "actual": [ "S-R9249" ] }, { "record_id": "R3919", "passed": true, "expected": [ "S-R3919" ], "actual": [ "S-R3919" ] }, { "record_id": "R2909", "passed": true, "expected": [ "S-R2909" ], "actual": [ "S-R2909", "U-R2909-3" ] }, { "record_id": "R4338", "passed": true, "expected": [ "S-R4338" ], "actual": [ "S-R4338", "U-R4338-2" ] }, { "record_id": "R9956", "passed": true, "expected": [ "S-R9956", "U-R9956-1" ], "actual": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R9815", "passed": true, "expected": [ "S-R2141", "S-R4738", "S-R9815" ], "actual": [ "S-R2141", "S-R4738", "S-R9815" ] } ], "source_sha256": "ce67e0718f3026eb2874d60176cdc3fc2c4395cef709ababd886f12325807ed5" } }, { "case_id": "sponsor-leads-cypress", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:23:57.212435+00:00", "completed_at": "2026-09-07T03:25:14.815735+00:00", "elapsed_seconds": 77.603, "cost_usd": 0.103005, "known_partial_cost_usd": 0.103005, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5468, "completion_tokens": 9617, "total_tokens": 15085, "cost": 0.103005, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7104, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.103005, "gateway_cost": 0.103005 }, "finish_reason": "stop", "elapsed_seconds": 77.584, "completed_at": "2026-09-07T03:25:14.814289+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sponsor-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-cypress", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 19, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R9249", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11000000, "expected": 11000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9249" ], "reported_sources": [ "S-R9249" ] }, { "record_id": "R3919", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "field": "firm_name", "actual": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10400000, "expected": 10400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3919" ], "reported_sources": [ "S-R3919" ] }, { "record_id": "R2909", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9800000, "expected": 9800000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2909" ], "reported_sources": [ "S-R2909" ] }, { "record_id": "R4338", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9200000, "expected": 9200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4338" ], "reported_sources": [ "S-R4338" ] }, { "record_id": "R9956", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "field": "firm_name", "actual": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8600000, "expected": 8600000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9956", "U-R9956-1" ], "reported_sources": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R2141", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11600000, "expected": 11600000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2141", "S-R9815" ], "reported_sources": [ "S-R2141", "S-R9815" ] } ], "decision_checks": [ { "record_id": "R9956", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3919", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5272", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R7976", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2395", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R4766", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8528", "actual": null, "expected": false, "passed": false, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4857", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3688", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R9249", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2909", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9289", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4738", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6278", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6741", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R9815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2141", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7602", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R4338", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1826", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-cypress", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:24:24.394790+00:00", "completed_at": "2026-09-07T03:25:31.191827+00:00", "elapsed_seconds": 66.797, "cost_usd": 0.211495, "known_partial_cost_usd": 0.211495, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8044, "completion_tokens": 6851, "total_tokens": 14895, "cost": 0.211495, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1594, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.211495, "gateway_cost": 0.211495 }, "finish_reason": "stop", "elapsed_seconds": 66.772, "completed_at": "2026-09-07T03:25:31.189890+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sponsor-leads-cypress", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-cypress", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R9249", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-30009", "expected": "CYPRESS-SPO-30009", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 6 Capital", "expected": "Cypress Square 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 11000000, "expected": 11000000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Cypress6", "expected": "Morgan Cypress6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-cypress.example", "expected": "person6@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9249" ], "reported_sources": [ "S-R9249" ] }, { "record_id": "R3919", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-21730", "expected": "CYPRESS-SPO-21730", "passed": true }, { "field": "firm_name", "actual": "Cypress Park 5 Capital", "expected": "Cypress Park 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 10400000, "expected": 10400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress5", "expected": "Alex Cypress5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3919" ], "reported_sources": [ "S-R3919" ] }, { "record_id": "R2909", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-81408", "expected": "CYPRESS-SPO-81408", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 4 Capital", "expected": "Cypress Crossing 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9800000, "expected": 9800000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Cypress4", "expected": "Taylor Cypress4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-cypress.example", "expected": "person4@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2909" ], "reported_sources": [ "S-R2909" ] }, { "record_id": "R4338", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-51531", "expected": "CYPRESS-SPO-51531", "passed": true }, { "field": "firm_name", "actual": "Cypress Square 3 Capital", "expected": "Cypress Square 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 9200000, "expected": 9200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Cypress3", "expected": "Jordan Cypress3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-cypress.example", "expected": "person3@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4338" ], "reported_sources": [ "S-R4338" ] }, { "record_id": "R9956", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-38583", "expected": "CYPRESS-SPO-38583", "passed": true }, { "field": "firm_name", "actual": "Cypress Point 2 Capital", "expected": "Cypress Point 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8600000, "expected": 8600000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Cypress", "expected": "Drew Cypress", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-cypress.example", "expected": "new.cfo@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9956", "U-R9956-1" ], "reported_sources": [ "S-R9956", "U-R9956-1" ] }, { "record_id": "R9815", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "CYPRESS-SPO-90059", "expected": "CYPRESS-SPO-90059", "passed": true }, { "field": "firm_name", "actual": "Cypress Crossing 1 Capital", "expected": "Cypress Crossing 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 8000000, "expected": 8000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Cypress1", "expected": "Alex Cypress1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-cypress.example", "expected": "person1@sponsor-leads-cypress.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9815" ], "reported_sources": [ "S-R9815" ] } ], "decision_checks": [ { "record_id": "R9956", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3919", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5272", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R7976", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R2395", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R4766", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8528", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R4857", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R3688", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R9249", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2909", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9289", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4738", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6278", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R6741", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R9815", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2141", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R7602", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R4338", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1826", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-dogwood", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:35:07.184Z", "observed_completed_at": "2026-09-07T02:40:11.966Z", "packet_sha256": "e4c1bd9feb9905b8e83f488ae56dd0c9fde161ebc4ddefa0db519aa72ca5fa24", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/dogwood_tampa_mf_sponsor_leads.csv", "sha256": "cd8b01f087cc73c7a7b1672a0bf2538b52e363067366f9ed751978511d6a8572", "bytes": 1118 }, { "path": "artifacts/dogwood_tampa_mf_sponsor_leads.xlsx", "sha256": "abc1645155a13f95e3e674772ca26be20f1b0f6f8c6ee7da209b5a18875c8ec7", "bytes": 5616 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.379625, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 92.53, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "78630138-9575-42e3-9eb3-19dfc0120a00", "session_id": "ff174183-09fa-47cf-9981-93b341c7ab0a", "started_at": "2026-09-06T22:36:00.700000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.22573875, 0.15388625 ], "total_cost_usd": 0.379625, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 92.53, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sponsor-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-dogwood", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 7, "eligible_count": 6, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 0.8333333333333334, "selection_precision": 0.8571428571428571, "correct_abstention": null, "field_accuracy": { "passed": 47, "total": 49 }, "evidence_accuracy": { "passed": 7, "total": 7 }, "candidate_accuracy": { "passed": 18, "total": 20 }, "eligible_recall": { "passed": 7, "total": 8 }, "ineligible_rejection": { "passed": 11, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R4246", "type": "ineligible_selection", "reasons": [ "Source published after as-of date" ] }, { "record_id": "R1890", "type": "material_field_error", "field": "transaction_size_usd", "actual": 40600000, "expected": 28000000 }, { "record_id": "R1890", "type": "material_field_error", "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-05" } ], "selected_checks": [ { "record_id": "R4246", "eligible": false, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-75075", "expected": "DOGWOOD-SPO-75075", "passed": true }, { "field": "firm_name", "actual": "Dogwood Grove 18 Capital", "expected": "Dogwood Grove 18 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34300000, "expected": 34300000, "passed": true }, { "field": "event_date", "actual": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "field": "contact_name", "actual": "Morgan Dogwood18", "expected": "Morgan Dogwood18", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person18@sponsor-leads-dogwood.example", "expected": "person18@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R4246" ], "reported_sources": [ "S-R4246" ] }, { "record_id": "R2197", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 38500000, "expected": 38500000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2197" ], "reported_sources": [ "S-R2197" ] }, { "record_id": "R9831", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 36400000, "expected": 36400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9831" ], "reported_sources": [ "S-R9831" ] }, { "record_id": "R3432", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34300000, "expected": 34300000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3432" ], "reported_sources": [ "S-R3432" ] }, { "record_id": "R1761", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "field": "firm_name", "actual": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 32200000, "expected": 32200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1761" ], "reported_sources": [ "S-R1761" ] }, { "record_id": "R9429", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 30100000, "expected": 30100000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9429", "U-R9429-1" ], "reported_sources": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R1890", "eligible": true, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "field": "firm_name", "actual": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 40600000, "expected": 28000000, "passed": false }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-05", "passed": false }, { "field": "contact_name", "actual": "Alex Dogwood1", "expected": "Alex Dogwood1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1890" ], "reported_sources": [ "S-R1890", "S-R1101", "S-R6695" ] } ], "decision_checks": [ { "record_id": "R9429", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R3088", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R1890", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3718", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R3432", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1135", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9543", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3924", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4246", "actual": true, "expected": false, "passed": false, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4968", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8500", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1658", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1101", "actual": false, "expected": true, "passed": false, "reasons": [] }, { "record_id": "R1761", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2197", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5255", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9831", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3779", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/dogwood_tampa_mf_sponsor_leads.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "R9831 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R4246", "R2197", "R9831", "R3432", "R1761", "R9429", "R1890" ], "answer_record_order": [ "R4246", "R2197", "R9831", "R3432", "R1761", "R9429", "R1890" ], "field_checks": [ { "record_id": "R4246", "field": "entity_id", "actual": "DOGWOOD-SPO-75075", "original": "DOGWOOD-SPO-75075", "expected": "DOGWOOD-SPO-75075", "passed": true }, { "record_id": "R4246", "field": "firm_name", "actual": "Dogwood Grove 18 Capital", "original": "Dogwood Grove 18 Capital", "expected": "Dogwood Grove 18 Capital", "passed": true }, { "record_id": "R4246", "field": "transaction_size_usd", "actual": "34300000", "original": "34300000", "expected": 34300000, "passed": true }, { "record_id": "R4246", "field": "event_date", "actual": "2026-07-22", "original": "2026-07-22", "expected": "2026-07-22", "passed": true }, { "record_id": "R4246", "field": "contact_name", "actual": "Morgan Dogwood18", "original": "Morgan Dogwood18", "expected": "Morgan Dogwood18", "passed": true }, { "record_id": "R4246", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R4246", "field": "contact_email", "actual": "person18@sponsor-leads-dogwood.example", "original": "person18@sponsor-leads-dogwood.example", "expected": "person18@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R2197", "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "original": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "record_id": "R2197", "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "original": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "record_id": "R2197", "field": "transaction_size_usd", "actual": "38500000", "original": "38500000", "expected": 38500000, "passed": true }, { "record_id": "R2197", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R2197", "field": "contact_name", "actual": "Morgan Dogwood6", "original": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "record_id": "R2197", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2197", "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "original": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R9831", "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "original": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "record_id": "R9831", "field": "firm_name", "actual": "Dogwood Square 5 Capital", "original": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "record_id": "R9831", "field": "transaction_size_usd", "actual": "36400000", "original": "36400000", "expected": 36400000, "passed": true }, { "record_id": "R9831", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R9831", "field": "contact_name", "actual": "Alex Dogwood5", "original": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "record_id": "R9831", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9831", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false }, { "record_id": "R3432", "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "original": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "record_id": "R3432", "field": "firm_name", "actual": "Dogwood Square 4 Capital", "original": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "record_id": "R3432", "field": "transaction_size_usd", "actual": "34300000", "original": "34300000", "expected": 34300000, "passed": true }, { "record_id": "R3432", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R3432", "field": "contact_name", "actual": "Taylor Dogwood4", "original": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "record_id": "R3432", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R3432", "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "original": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R1761", "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "original": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "record_id": "R1761", "field": "firm_name", "actual": "Dogwood Point 3 Capital", "original": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "record_id": "R1761", "field": "transaction_size_usd", "actual": "32200000", "original": "32200000", "expected": 32200000, "passed": true }, { "record_id": "R1761", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1761", "field": "contact_name", "actual": "Jordan Dogwood3", "original": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "record_id": "R1761", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1761", "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "original": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R9429", "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "original": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "record_id": "R9429", "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "original": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "record_id": "R9429", "field": "transaction_size_usd", "actual": "30100000", "original": "30100000", "expected": 30100000, "passed": true }, { "record_id": "R9429", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9429", "field": "contact_name", "actual": "Drew Dogwood", "original": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "record_id": "R9429", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9429", "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "original": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R1890", "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "original": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "record_id": "R1890", "field": "firm_name", "actual": "Dogwood Park 1 Capital", "original": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "record_id": "R1890", "field": "transaction_size_usd", "actual": "40600000", "original": "40600000", "expected": 40600000, "passed": true }, { "record_id": "R1890", "field": "event_date", "actual": "2026-02-11", "original": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "record_id": "R1890", "field": "contact_name", "actual": "Alex Dogwood1", "original": "Alex Dogwood1", "expected": "Alex Dogwood1", "passed": true }, { "record_id": "R1890", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1890", "field": "contact_email", "actual": "person1@sponsor-leads-dogwood.example", "original": "person1@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R4246", "passed": true, "expected": [ "S-R4246" ], "actual": [ "S-R4246" ] }, { "record_id": "R2197", "passed": true, "expected": [ "S-R2197" ], "actual": [ "S-R2197" ] }, { "record_id": "R9831", "passed": true, "expected": [ "S-R9831" ], "actual": [ "S-R9831" ] }, { "record_id": "R3432", "passed": true, "expected": [ "S-R3432" ], "actual": [ "S-R3432" ] }, { "record_id": "R1761", "passed": true, "expected": [ "S-R1761" ], "actual": [ "S-R1761" ] }, { "record_id": "R9429", "passed": true, "expected": [ "S-R9429", "U-R9429-1" ], "actual": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R1890", "passed": true, "expected": [ "S-R1101", "S-R1890", "S-R6695" ], "actual": [ "S-R1101", "S-R1890", "S-R6695" ] } ], "source_sha256": "cd8b01f087cc73c7a7b1672a0bf2538b52e363067366f9ed751978511d6a8572" } }, { "case_id": "sponsor-leads-dogwood", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:25:14.817141+00:00", "completed_at": "2026-09-07T03:27:50.109646+00:00", "elapsed_seconds": 155.292, "cost_usd": 0.17334175000000002, "known_partial_cost_usd": 0.17334175000000002, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6271, "completion_tokens": 8128, "total_tokens": 14399, "cost": 0.08911875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7680, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.08911875, "gateway_cost": 0.08911875 }, "finish_reason": "tool_calls", "elapsed_seconds": 72.417, "completed_at": "2026-09-07T03:26:27.253276+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6684, "completion_tokens": 8278, "total_tokens": 14962, "cost": 0.084223, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6144, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6272, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.084223, "gateway_cost": 0.084223 }, "finish_reason": "tool_calls", "elapsed_seconds": 82.851, "completed_at": "2026-09-07T03:27:50.107042+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "265811ee87d16f19b7e3e3c88adaedab1177e880ed00c1ca14ffdfa9d3992451", "bytes": 936 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-dogwood", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 0.8333333333333334, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 5, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2197", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 38500000, "expected": 38500000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2197" ], "reported_sources": [ "S-R2197" ] }, { "record_id": "R9831", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 36400000, "expected": 36400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9831" ], "reported_sources": [ "S-R9831" ] }, { "record_id": "R3432", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34300000, "expected": 34300000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3432" ], "reported_sources": [ "S-R3432" ] }, { "record_id": "R1761", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "field": "firm_name", "actual": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 32200000, "expected": 32200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1761" ], "reported_sources": [ "S-R1761" ] }, { "record_id": "R9429", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 30100000, "expected": 30100000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9429", "U-R9429-1" ], "reported_sources": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R6695", "eligible": true, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "field": "firm_name", "actual": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 40600000, "expected": 40600000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood1", "expected": "Alex Dogwood1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": false, "required_sources": [ "S-R1101", "S-R6695" ], "reported_sources": [ "S-R6695", "S-R1890" ] } ], "decision_checks": [ { "record_id": "R9429", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R3088", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R1890", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3718", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R3432", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1135", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9543", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3924", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4246", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4968", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8500", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1658", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1101", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1761", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2197", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5255", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9831", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3779", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R9831 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2197", "R9831", "R3432", "R1761", "R9429", "R6695" ], "answer_record_order": [ "R2197", "R9831", "R3432", "R1761", "R9429", "R6695" ], "field_checks": [ { "record_id": "R2197", "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "original": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "record_id": "R2197", "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "original": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "record_id": "R2197", "field": "transaction_size_usd", "actual": "38500000", "original": "38500000", "expected": 38500000, "passed": true }, { "record_id": "R2197", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R2197", "field": "contact_name", "actual": "Morgan Dogwood6", "original": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "record_id": "R2197", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2197", "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "original": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R9831", "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "original": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "record_id": "R9831", "field": "firm_name", "actual": "Dogwood Square 5 Capital", "original": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "record_id": "R9831", "field": "transaction_size_usd", "actual": "36400000", "original": "36400000", "expected": 36400000, "passed": true }, { "record_id": "R9831", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R9831", "field": "contact_name", "actual": "Alex Dogwood5", "original": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "record_id": "R9831", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9831", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R3432", "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "original": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "record_id": "R3432", "field": "firm_name", "actual": "Dogwood Square 4 Capital", "original": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "record_id": "R3432", "field": "transaction_size_usd", "actual": "34300000", "original": "34300000", "expected": 34300000, "passed": true }, { "record_id": "R3432", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R3432", "field": "contact_name", "actual": "Taylor Dogwood4", "original": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "record_id": "R3432", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R3432", "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "original": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R1761", "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "original": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "record_id": "R1761", "field": "firm_name", "actual": "Dogwood Point 3 Capital", "original": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "record_id": "R1761", "field": "transaction_size_usd", "actual": "32200000", "original": "32200000", "expected": 32200000, "passed": true }, { "record_id": "R1761", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1761", "field": "contact_name", "actual": "Jordan Dogwood3", "original": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "record_id": "R1761", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1761", "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "original": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R9429", "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "original": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "record_id": "R9429", "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "original": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "record_id": "R9429", "field": "transaction_size_usd", "actual": "30100000", "original": "30100000", "expected": 30100000, "passed": true }, { "record_id": "R9429", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9429", "field": "contact_name", "actual": "Drew Dogwood", "original": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "record_id": "R9429", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9429", "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "original": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R6695", "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "original": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "record_id": "R6695", "field": "firm_name", "actual": "Dogwood Park 1 Capital", "original": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "record_id": "R6695", "field": "transaction_size_usd", "actual": "40600000", "original": "40600000", "expected": 40600000, "passed": true }, { "record_id": "R6695", "field": "event_date", "actual": "2026-02-11", "original": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "record_id": "R6695", "field": "contact_name", "actual": "Alex Dogwood1", "original": "Alex Dogwood1", "expected": "Alex Dogwood1", "passed": true }, { "record_id": "R6695", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R6695", "field": "contact_email", "actual": "person1@sponsor-leads-dogwood.example", "original": "person1@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R2197", "passed": true, "expected": [ "S-R2197" ], "actual": [ "S-R2197" ] }, { "record_id": "R9831", "passed": true, "expected": [ "S-R9831" ], "actual": [ "S-R9831" ] }, { "record_id": "R3432", "passed": true, "expected": [ "S-R3432" ], "actual": [ "S-R3432" ] }, { "record_id": "R1761", "passed": true, "expected": [ "S-R1761" ], "actual": [ "S-R1761" ] }, { "record_id": "R9429", "passed": true, "expected": [ "S-R9429", "U-R9429-1" ], "actual": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R6695", "passed": true, "expected": [ "S-R1890", "S-R6695" ], "actual": [ "S-R1890", "S-R6695" ] } ], "source_sha256": "265811ee87d16f19b7e3e3c88adaedab1177e880ed00c1ca14ffdfa9d3992451" } }, { "case_id": "sponsor-leads-dogwood", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:25:31.193264+00:00", "completed_at": "2026-09-07T03:26:43.356756+00:00", "elapsed_seconds": 72.163, "cost_usd": 0.29317499999999996, "known_partial_cost_usd": 0.29317499999999996, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 1, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10455, "completion_tokens": 2375, "total_tokens": 12830, "cost": 0.11165, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1557, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11165, "gateway_cost": 0.11165 }, "finish_reason": "tool_calls", "elapsed_seconds": 23.971, "completed_at": "2026-09-07T03:25:55.184353+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 11365, "completion_tokens": 4988, "total_tokens": 16353, "cost": 0.181525, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 601, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.181525, "gateway_cost": 0.181525 }, "finish_reason": "stop", "elapsed_seconds": 48.167, "completed_at": "2026-09-07T03:26:43.354818+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "254a54be388a03e6243fdf37d96d0218edc5710f75da964bf1c83aca245f23df", "bytes": 1220 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-dogwood", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 0.8333333333333334, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 40, "total": 42 }, "evidence_accuracy": { "passed": 5, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [ { "record_id": "R6695", "type": "material_field_error", "field": "contact_name", "actual": "Jordan Dogwood7", "expected": "Alex Dogwood1" }, { "record_id": "R6695", "type": "material_field_error", "field": "contact_email", "actual": "person7@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example" } ], "selected_checks": [ { "record_id": "R2197", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 38500000, "expected": 38500000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2197" ], "reported_sources": [ "S-R2197" ] }, { "record_id": "R9831", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 36400000, "expected": 36400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9831" ], "reported_sources": [ "S-R9831" ] }, { "record_id": "R3432", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34300000, "expected": 34300000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3432" ], "reported_sources": [ "S-R3432" ] }, { "record_id": "R1761", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "field": "firm_name", "actual": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 32200000, "expected": 32200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1761" ], "reported_sources": [ "S-R1761" ] }, { "record_id": "R9429", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 30100000, "expected": 30100000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9429", "U-R9429-1" ], "reported_sources": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R6695", "eligible": true, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "field": "firm_name", "actual": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 40600000, "expected": 40600000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Jordan Dogwood7", "expected": "Alex Dogwood1", "passed": false }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person7@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": false } ], "evidence_passed": false, "required_sources": [ "S-R1101", "S-R6695" ], "reported_sources": [ "S-R6695", "S-R1890" ] } ], "decision_checks": [ { "record_id": "R9429", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R3088", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R1890", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3718", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R3432", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1135", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9543", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3924", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4246", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4968", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8500", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1658", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1101", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1761", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2197", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5255", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9831", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3779", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [ { "record_id": "R6695", "field": "firm_name", "original": "Dogwood Park 1 Capital (transaction closed by property SPV Dogwood Crossing 7 Capital)", "normalized": "Dogwood Park 1 Capital", "reason": "Source-proven parent label with accurate SPV annotation" } ] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": false }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [ "1 blank unknown annotations were caused by the benchmark CSV writer after explicit model-supplied JSON nulls; these are harness delivery failures, not model factual errors.", "R9831 / contact_email: artifact differs from answer or lacks explicit unknown" ], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2197", "R9831", "R3432", "R1761", "R9429", "R6695" ], "answer_record_order": [ "R2197", "R9831", "R3432", "R1761", "R9429", "R6695" ], "field_checks": [ { "record_id": "R2197", "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "original": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "record_id": "R2197", "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "original": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "record_id": "R2197", "field": "transaction_size_usd", "actual": "38500000", "original": "38500000", "expected": 38500000, "passed": true }, { "record_id": "R2197", "field": "event_date", "actual": "2026-07-10", "original": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "record_id": "R2197", "field": "contact_name", "actual": "Morgan Dogwood6", "original": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "record_id": "R2197", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R2197", "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "original": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R9831", "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "original": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "record_id": "R9831", "field": "firm_name", "actual": "Dogwood Square 5 Capital", "original": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "record_id": "R9831", "field": "transaction_size_usd", "actual": "36400000", "original": "36400000", "expected": 36400000, "passed": true }, { "record_id": "R9831", "field": "event_date", "actual": "2026-06-09", "original": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "record_id": "R9831", "field": "contact_name", "actual": "Alex Dogwood5", "original": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "record_id": "R9831", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9831", "field": "contact_email", "actual": "", "original": "", "expected": null, "passed": false, "attribution": "benchmark_csv_serializer", "attribution_detail": "Model supplied explicit JSON null; the frozen CSV writer serialized it as a blank cell.", "tool_evidence": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-dogwood/turn-01/tool-01.json" }, { "record_id": "R3432", "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "original": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "record_id": "R3432", "field": "firm_name", "actual": "Dogwood Square 4 Capital", "original": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "record_id": "R3432", "field": "transaction_size_usd", "actual": "34300000", "original": "34300000", "expected": 34300000, "passed": true }, { "record_id": "R3432", "field": "event_date", "actual": "2026-05-08", "original": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "record_id": "R3432", "field": "contact_name", "actual": "Taylor Dogwood4", "original": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "record_id": "R3432", "field": "contact_role", "actual": "Head of Acquisitions", "original": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "record_id": "R3432", "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "original": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R1761", "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "original": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "record_id": "R1761", "field": "firm_name", "actual": "Dogwood Point 3 Capital", "original": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "record_id": "R1761", "field": "transaction_size_usd", "actual": "32200000", "original": "32200000", "expected": 32200000, "passed": true }, { "record_id": "R1761", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R1761", "field": "contact_name", "actual": "Jordan Dogwood3", "original": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "record_id": "R1761", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R1761", "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "original": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R9429", "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "original": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "record_id": "R9429", "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "original": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "record_id": "R9429", "field": "transaction_size_usd", "actual": "30100000", "original": "30100000", "expected": 30100000, "passed": true }, { "record_id": "R9429", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R9429", "field": "contact_name", "actual": "Drew Dogwood", "original": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "record_id": "R9429", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9429", "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "original": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true }, { "record_id": "R6695", "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "original": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "record_id": "R6695", "field": "firm_name", "actual": "Dogwood Park 1 Capital", "original": "Dogwood Park 1 Capital (parent of SPV Dogwood Crossing 7 Capital)", "expected": "Dogwood Park 1 Capital", "passed": true }, { "record_id": "R6695", "field": "transaction_size_usd", "actual": "40600000", "original": "40600000", "expected": 40600000, "passed": true }, { "record_id": "R6695", "field": "event_date", "actual": "2026-02-11", "original": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "record_id": "R6695", "field": "contact_name", "actual": "Jordan Dogwood7", "original": "Jordan Dogwood7", "expected": "Jordan Dogwood7", "passed": true }, { "record_id": "R6695", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R6695", "field": "contact_email", "actual": "person7@sponsor-leads-dogwood.example", "original": "person7@sponsor-leads-dogwood.example", "expected": "person7@sponsor-leads-dogwood.example", "passed": true } ], "harness_induced_null_annotations": 1, "source_checks": [ { "record_id": "R2197", "passed": true, "expected": [ "S-R2197" ], "actual": [ "S-R2197" ] }, { "record_id": "R9831", "passed": true, "expected": [ "S-R9831" ], "actual": [ "S-R9831" ] }, { "record_id": "R3432", "passed": true, "expected": [ "S-R3432" ], "actual": [ "S-R3432", "U-R3432-3" ] }, { "record_id": "R1761", "passed": true, "expected": [ "S-R1761" ], "actual": [ "S-R1761", "U-R1761-2" ] }, { "record_id": "R9429", "passed": true, "expected": [ "S-R9429", "U-R9429-1" ], "actual": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R6695", "passed": true, "expected": [ "S-R1890", "S-R6695" ], "actual": [ "S-R1101", "S-R1890", "S-R6695" ] } ], "source_sha256": "254a54be388a03e6243fdf37d96d0218edc5710f75da964bf1c83aca245f23df" } }, { "case_id": "sponsor-leads-dogwood", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:26:43.357662+00:00", "completed_at": "2026-09-07T03:28:02.611152+00:00", "elapsed_seconds": 79.253, "cost_usd": 0.11206625, "known_partial_cost_usd": 0.11206625, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5525, "completion_tokens": 10516, "total_tokens": 16041, "cost": 0.11206625, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 7872, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.11206625, "gateway_cost": 0.11206625 }, "finish_reason": "stop", "elapsed_seconds": 79.233, "completed_at": "2026-09-07T03:28:02.609641+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sponsor-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-dogwood", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 5, "eligible_yield": 1.0, "verified_yield": 0.8333333333333334, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 5, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2197", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 38500000, "expected": 38500000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2197" ], "reported_sources": [ "S-R2197" ] }, { "record_id": "R9831", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 36400000, "expected": 36400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9831" ], "reported_sources": [ "S-R9831" ] }, { "record_id": "R3432", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34300000, "expected": 34300000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3432" ], "reported_sources": [ "S-R3432" ] }, { "record_id": "R1761", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "field": "firm_name", "actual": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 32200000, "expected": 32200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1761" ], "reported_sources": [ "S-R1761" ] }, { "record_id": "R9429", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 30100000, "expected": 30100000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9429", "U-R9429-1" ], "reported_sources": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R6695", "eligible": true, "duplicate": false, "verified": false, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "field": "firm_name", "actual": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 40600000, "expected": 40600000, "passed": true }, { "field": "event_date", "actual": "2026-02-11", "expected": "2026-02-11", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood1", "expected": "Alex Dogwood1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": false, "required_sources": [ "S-R1101", "S-R6695" ], "reported_sources": [ "S-R6695", "S-R1890" ] } ], "decision_checks": [ { "record_id": "R9429", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R3088", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R1890", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3718", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R3432", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1135", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9543", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3924", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4246", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4968", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8500", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1658", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1101", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1761", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2197", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5255", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9831", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3779", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-dogwood", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:27:50.110566+00:00", "completed_at": "2026-09-07T03:29:14.225751+00:00", "elapsed_seconds": 84.115, "cost_usd": 0.2234, "known_partial_cost_usd": 0.2234, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8170, "completion_tokens": 7302, "total_tokens": 15472, "cost": 0.2234, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1835, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.2234, "gateway_cost": 0.2234 }, "finish_reason": "stop", "elapsed_seconds": 84.095, "completed_at": "2026-09-07T03:29:14.223622+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sponsor-leads-dogwood", "expected_target": 6, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-dogwood", "task": "sponsor_leads", "target_count": 6, "available_unique": 6, "returned_count": 6, "eligible_count": 6, "verified_count": 6, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 42, "total": 42 }, "evidence_accuracy": { "passed": 6, "total": 6 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 8, "total": 8 }, "ineligible_rejection": { "passed": 12, "total": 12 }, "ranking_pair_agreement": { "passed": 15, "total": 15 }, "material_errors": [], "selected_checks": [ { "record_id": "R2197", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-12301", "expected": "DOGWOOD-SPO-12301", "passed": true }, { "field": "firm_name", "actual": "Dogwood Grove 6 Capital", "expected": "Dogwood Grove 6 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 38500000, "expected": 38500000, "passed": true }, { "field": "event_date", "actual": "2026-07-10", "expected": "2026-07-10", "passed": true }, { "field": "contact_name", "actual": "Morgan Dogwood6", "expected": "Morgan Dogwood6", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person6@sponsor-leads-dogwood.example", "expected": "person6@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2197" ], "reported_sources": [ "S-R2197" ] }, { "record_id": "R9831", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-79419", "expected": "DOGWOOD-SPO-79419", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 5 Capital", "expected": "Dogwood Square 5 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 36400000, "expected": 36400000, "passed": true }, { "field": "event_date", "actual": "2026-06-09", "expected": "2026-06-09", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood5", "expected": "Alex Dogwood5", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": null, "expected": null, "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9831" ], "reported_sources": [ "S-R9831" ] }, { "record_id": "R3432", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-28763", "expected": "DOGWOOD-SPO-28763", "passed": true }, { "field": "firm_name", "actual": "Dogwood Square 4 Capital", "expected": "Dogwood Square 4 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 34300000, "expected": 34300000, "passed": true }, { "field": "event_date", "actual": "2026-05-08", "expected": "2026-05-08", "passed": true }, { "field": "contact_name", "actual": "Taylor Dogwood4", "expected": "Taylor Dogwood4", "passed": true }, { "field": "contact_role", "actual": "Head of Acquisitions", "expected": "Head of Acquisitions", "passed": true }, { "field": "contact_email", "actual": "person4@sponsor-leads-dogwood.example", "expected": "person4@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3432" ], "reported_sources": [ "S-R3432" ] }, { "record_id": "R1761", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-33898", "expected": "DOGWOOD-SPO-33898", "passed": true }, { "field": "firm_name", "actual": "Dogwood Point 3 Capital", "expected": "Dogwood Point 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 32200000, "expected": 32200000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Dogwood3", "expected": "Jordan Dogwood3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-dogwood.example", "expected": "person3@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1761" ], "reported_sources": [ "S-R1761" ] }, { "record_id": "R9429", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-39382", "expected": "DOGWOOD-SPO-39382", "passed": true }, { "field": "firm_name", "actual": "Dogwood Crossing 2 Capital", "expected": "Dogwood Crossing 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 30100000, "expected": 30100000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Dogwood", "expected": "Drew Dogwood", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-dogwood.example", "expected": "new.cfo@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9429", "U-R9429-1" ], "reported_sources": [ "S-R9429", "U-R9429-1" ] }, { "record_id": "R1890", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "DOGWOOD-SPO-18376", "expected": "DOGWOOD-SPO-18376", "passed": true }, { "field": "firm_name", "actual": "Dogwood Park 1 Capital", "expected": "Dogwood Park 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 28000000, "expected": 28000000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Dogwood1", "expected": "Alex Dogwood1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-dogwood.example", "expected": "person1@sponsor-leads-dogwood.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R1890" ], "reported_sources": [ "S-R1890", "S-R1101" ] } ], "decision_checks": [ { "record_id": "R9429", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R6695", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1218", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R3088", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R1890", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3718", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R3432", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1135", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R9543", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3924", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4246", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R4968", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8500", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1658", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R1101", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R1761", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R2197", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R5255", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9831", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3779", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-elm", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:38:08.575Z", "observed_completed_at": "2026-09-07T02:44:54.727Z", "packet_sha256": "22cd32b018343d76823fb19e007681d0da78f7df743185bebcbe98862039edff", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/sponsor_leads_comp_grid.xlsx", "sha256": "8212333354f26f990fbeb4c146e6e2a884557cb4df725b1505d74620c0753ea8", "bytes": 8300 }, { "path": "artifacts/sponsor_leads_prospects.csv", "sha256": "5ea3d66c8ba76f340e6e06a6741a8869da000091b82429e62d4505f5f2abf60f", "bytes": 910 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.6938044999999999, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 155.65, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "677f4db1-e303-4b16-acd6-eb2593c31669", "session_id": "e0771ea2-7242-4981-994c-bc4783e7ba9c", "started_at": "2026-09-06T22:40:12.694000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.51335, 0.1804545 ], "total_cost_usd": 0.6938044999999999, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 155.65, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sponsor-leads-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-elm", "task": "sponsor_leads", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R2814", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "field": "firm_name", "actual": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2814" ], "reported_sources": [ "S-R2814" ] }, { "record_id": "R8068", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "field": "firm_name", "actual": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 15480000, "expected": 15480000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8068", "U-R8068-1" ], "reported_sources": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R3531", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "field": "firm_name", "actual": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3531" ], "reported_sources": [ "S-R3531" ] } ], "decision_checks": [ { "record_id": "R3531", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9425", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R4981", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9588", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4183", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6876", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3693", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3654", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R2866", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2827", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6495", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4894", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9706", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R9106", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2109", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3292", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2814", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4621", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R8068", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3284", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/sponsor_leads_prospects.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2814", "R8068", "R3531" ], "answer_record_order": [ "R2814", "R8068", "R3531" ], "field_checks": [ { "record_id": "R2814", "field": "entity_id", "actual": "ELM-SPO-56859", "original": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "record_id": "R2814", "field": "firm_name", "actual": "Elm Grove 3 Capital", "original": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "record_id": "R2814", "field": "transaction_size_usd", "actual": "16560000", "original": "16560000", "expected": 16560000, "passed": true }, { "record_id": "R2814", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2814", "field": "contact_name", "actual": "Jordan Elm3", "original": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "record_id": "R2814", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2814", "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "original": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true }, { "record_id": "R8068", "field": "entity_id", "actual": "ELM-SPO-69766", "original": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "record_id": "R8068", "field": "firm_name", "actual": "Elm Square 2 Capital", "original": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "record_id": "R8068", "field": "transaction_size_usd", "actual": "15480000", "original": "15480000", "expected": 15480000, "passed": true }, { "record_id": "R8068", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8068", "field": "contact_name", "actual": "Drew Elm", "original": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "record_id": "R8068", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R8068", "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "original": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true }, { "record_id": "R3531", "field": "entity_id", "actual": "ELM-SPO-34913", "original": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "record_id": "R3531", "field": "firm_name", "actual": "Elm Point 1 Capital", "original": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "record_id": "R3531", "field": "transaction_size_usd", "actual": "14400000", "original": "14400000", "expected": 14400000, "passed": true }, { "record_id": "R3531", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R3531", "field": "contact_name", "actual": "Alex Elm1", "original": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "record_id": "R3531", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R3531", "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "original": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2814", "passed": true, "expected": [ "S-R2814" ], "actual": [ "S-R2814" ] }, { "record_id": "R8068", "passed": true, "expected": [ "S-R8068", "U-R8068-1" ], "actual": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R3531", "passed": true, "expected": [ "S-R3531" ], "actual": [ "S-R3531" ] } ], "source_sha256": "5ea3d66c8ba76f340e6e06a6741a8869da000091b82429e62d4505f5f2abf60f" } }, { "case_id": "sponsor-leads-elm", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:27:50.877316+00:00", "completed_at": "2026-09-07T03:31:23.944450+00:00", "elapsed_seconds": 213.066, "cost_usd": 0.27319225, "known_partial_cost_usd": 0.27319225, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 4, "tool_calls": 4, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6470, "completion_tokens": 6317, "total_tokens": 12787, "cost": 0.0712575, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6080, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0712575, "gateway_cost": 0.0712575 }, "finish_reason": "tool_calls", "elapsed_seconds": 49.188, "completed_at": "2026-09-07T03:28:40.080789+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6707, "completion_tokens": 5616, "total_tokens": 12323, "cost": 0.05734375, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6400, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5376, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.05734375, "gateway_cost": 0.05734375 }, "finish_reason": "tool_calls", "elapsed_seconds": 44.413, "completed_at": "2026-09-07T03:29:24.499208+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6949, "completion_tokens": 6669, "total_tokens": 13618, "cost": 0.06788825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6656, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6400, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06788825, "gateway_cost": 0.06788825 }, "finish_reason": "tool_calls", "elapsed_seconds": 53.241, "completed_at": "2026-09-07T03:30:17.743767+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 7191, "completion_tokens": 7549, "total_tokens": 14740, "cost": 0.07670275, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6912, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5760, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.07670275, "gateway_cost": 0.07670275 }, "finish_reason": "tool_calls", "elapsed_seconds": 66.195, "completed_at": "2026-09-07T03:31:23.942142+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "b3b14c86d6a7055829be8bb3f96180f4532814b3cc8c314434deec769ac538fa", "bytes": 477 }, { "path": "artifacts/prospects-v2.csv", "sha256": "0be393a6dfff27fa64abfbe7268c7214f6f173b1b477aaaabb5abf369af471a7", "bytes": 485 }, { "path": "artifacts/prospects-v3.csv", "sha256": "0be393a6dfff27fa64abfbe7268c7214f6f173b1b477aaaabb5abf369af471a7", "bytes": 485 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-elm", "task": "sponsor_leads", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R2814", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "field": "firm_name", "actual": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2814" ], "reported_sources": [ "S-R2814" ] }, { "record_id": "R8068", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "field": "firm_name", "actual": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 15480000, "expected": 15480000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8068", "U-R8068-1" ], "reported_sources": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R9588", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "field": "firm_name", "actual": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9588" ], "reported_sources": [ "S-R9588", "S-R3531" ] } ], "decision_checks": [ { "record_id": "R3531", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9425", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R4981", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9588", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4183", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6876", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3693", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3654", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R2866", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2827", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6495", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4894", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9706", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R9106", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2109", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3292", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2814", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4621", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R8068", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3284", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v3.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2814", "R8068", "R9588" ], "answer_record_order": [ "R2814", "R8068", "R9588" ], "field_checks": [ { "record_id": "R2814", "field": "entity_id", "actual": "ELM-SPO-56859", "original": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "record_id": "R2814", "field": "firm_name", "actual": "Elm Grove 3 Capital", "original": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "record_id": "R2814", "field": "transaction_size_usd", "actual": "16560000", "original": "16560000", "expected": 16560000, "passed": true }, { "record_id": "R2814", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2814", "field": "contact_name", "actual": "Jordan Elm3", "original": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "record_id": "R2814", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2814", "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "original": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true }, { "record_id": "R8068", "field": "entity_id", "actual": "ELM-SPO-69766", "original": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "record_id": "R8068", "field": "firm_name", "actual": "Elm Square 2 Capital", "original": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "record_id": "R8068", "field": "transaction_size_usd", "actual": "15480000", "original": "15480000", "expected": 15480000, "passed": true }, { "record_id": "R8068", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8068", "field": "contact_name", "actual": "Drew Elm", "original": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "record_id": "R8068", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R8068", "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "original": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true }, { "record_id": "R9588", "field": "entity_id", "actual": "ELM-SPO-34913", "original": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "record_id": "R9588", "field": "firm_name", "actual": "Elm Point 1 Capital", "original": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "record_id": "R9588", "field": "transaction_size_usd", "actual": "14400000", "original": "14400000", "expected": 14400000, "passed": true }, { "record_id": "R9588", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R9588", "field": "contact_name", "actual": "Alex Elm1", "original": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "record_id": "R9588", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9588", "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "original": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2814", "passed": true, "expected": [ "S-R2814" ], "actual": [ "S-R2814" ] }, { "record_id": "R8068", "passed": true, "expected": [ "S-R8068", "U-R8068-1" ], "actual": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R9588", "passed": true, "expected": [ "S-R3531", "S-R9588" ], "actual": [ "S-R3531", "S-R9588" ] } ], "source_sha256": "0be393a6dfff27fa64abfbe7268c7214f6f173b1b477aaaabb5abf369af471a7" } }, { "case_id": "sponsor-leads-elm", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:28:02.612025+00:00", "completed_at": "2026-09-07T03:29:04.283884+00:00", "elapsed_seconds": 61.672, "cost_usd": 0.260335, "known_partial_cost_usd": 0.260335, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 1, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10579, "completion_tokens": 1580, "total_tokens": 12159, "cost": 0.092395, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1179, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.092395, "gateway_cost": 0.092395 }, "finish_reason": "tool_calls", "elapsed_seconds": 16.979, "completed_at": "2026-09-07T03:28:19.608952+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 11048, "completion_tokens": 4508, "total_tokens": 15556, "cost": 0.16794, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 888, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.16794, "gateway_cost": 0.16794 }, "finish_reason": "stop", "elapsed_seconds": 44.67, "completed_at": "2026-09-07T03:29:04.281984+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "7512f3d9680d75a88ceb6047970108257024e453a4c0ec85abc5f8b6d57d7a51", "bytes": 550 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-elm", "task": "sponsor_leads", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R2814", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "field": "firm_name", "actual": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2814" ], "reported_sources": [ "S-R2814" ] }, { "record_id": "R8068", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "field": "firm_name", "actual": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 15480000, "expected": 15480000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8068", "U-R8068-1" ], "reported_sources": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R9588", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "field": "firm_name", "actual": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R9588" ], "reported_sources": [ "S-R9588", "S-R3531" ] } ], "decision_checks": [ { "record_id": "R3531", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9425", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R4981", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9588", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4183", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6876", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3693", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3654", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R2866", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2827", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6495", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4894", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9706", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R9106", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2109", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3292", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2814", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4621", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R8068", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3284", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [ "R2814", "R8068", "R9588" ], "answer_record_order": [ "R2814", "R8068", "R9588" ], "field_checks": [ { "record_id": "R2814", "field": "entity_id", "actual": "ELM-SPO-56859", "original": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "record_id": "R2814", "field": "firm_name", "actual": "Elm Grove 3 Capital", "original": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "record_id": "R2814", "field": "transaction_size_usd", "actual": "16560000", "original": "16560000", "expected": 16560000, "passed": true }, { "record_id": "R2814", "field": "event_date", "actual": "2026-04-07", "original": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "record_id": "R2814", "field": "contact_name", "actual": "Jordan Elm3", "original": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "record_id": "R2814", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R2814", "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "original": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true }, { "record_id": "R8068", "field": "entity_id", "actual": "ELM-SPO-69766", "original": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "record_id": "R8068", "field": "firm_name", "actual": "Elm Square 2 Capital", "original": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "record_id": "R8068", "field": "transaction_size_usd", "actual": "15480000", "original": "15480000", "expected": 15480000, "passed": true }, { "record_id": "R8068", "field": "event_date", "actual": "2026-03-06", "original": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "record_id": "R8068", "field": "contact_name", "actual": "Drew Elm", "original": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "record_id": "R8068", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R8068", "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "original": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true }, { "record_id": "R9588", "field": "entity_id", "actual": "ELM-SPO-34913", "original": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "record_id": "R9588", "field": "firm_name", "actual": "Elm Point 1 Capital", "original": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "record_id": "R9588", "field": "transaction_size_usd", "actual": "14400000", "original": "14400000", "expected": 14400000, "passed": true }, { "record_id": "R9588", "field": "event_date", "actual": "2026-02-05", "original": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "record_id": "R9588", "field": "contact_name", "actual": "Alex Elm1", "original": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "record_id": "R9588", "field": "contact_role", "actual": "CFO", "original": "CFO", "expected": "CFO", "passed": true }, { "record_id": "R9588", "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "original": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "harness_induced_null_annotations": 0, "source_checks": [ { "record_id": "R2814", "passed": true, "expected": [ "S-R2814" ], "actual": [ "S-R2814" ] }, { "record_id": "R8068", "passed": true, "expected": [ "S-R8068", "U-R8068-1" ], "actual": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R9588", "passed": true, "expected": [ "S-R3531", "S-R9588" ], "actual": [ "S-R3531", "S-R9588" ] } ], "source_sha256": "7512f3d9680d75a88ceb6047970108257024e453a4c0ec85abc5f8b6d57d7a51" } }, { "case_id": "sponsor-leads-elm", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:29:04.284866+00:00", "completed_at": "2026-09-07T03:30:13.688419+00:00", "elapsed_seconds": 69.403, "cost_usd": 0.074875, "known_partial_cost_usd": 0.074875, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5724, "completion_tokens": 6772, "total_tokens": 12496, "cost": 0.074875, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 4736, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.074875, "gateway_cost": 0.074875 }, "finish_reason": "stop", "elapsed_seconds": 69.384, "completed_at": "2026-09-07T03:30:13.687121+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sponsor-leads-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-elm", "task": "sponsor_leads", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R2814", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "field": "firm_name", "actual": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2814" ], "reported_sources": [ "S-R2814" ] }, { "record_id": "R8068", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "field": "firm_name", "actual": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 15480000, "expected": 15480000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8068", "U-R8068-1" ], "reported_sources": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R3531", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "field": "firm_name", "actual": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3531" ], "reported_sources": [ "S-R3531" ] } ], "decision_checks": [ { "record_id": "R3531", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9425", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R4981", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9588", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4183", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6876", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3693", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3654", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R2866", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2827", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6495", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4894", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9706", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R9106", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2109", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3292", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2814", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4621", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R8068", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3284", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-elm", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:29:14.226676+00:00", "completed_at": "2026-09-07T03:30:07.581916+00:00", "elapsed_seconds": 53.355, "cost_usd": 0.174295, "known_partial_cost_usd": 0.174295, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8294, "completion_tokens": 5313, "total_tokens": 13607, "cost": 0.174295, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1195, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.174295, "gateway_cost": 0.174295 }, "finish_reason": "stop", "elapsed_seconds": 53.335, "completed_at": "2026-09-07T03:30:07.578954+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sponsor-leads-elm", "expected_target": 3, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-elm", "task": "sponsor_leads", "target_count": 3, "available_unique": 3, "returned_count": 3, "eligible_count": 3, "verified_count": 3, "eligible_yield": 1.0, "verified_yield": 1.0, "selection_precision": 1.0, "correct_abstention": null, "field_accuracy": { "passed": 21, "total": 21 }, "evidence_accuracy": { "passed": 3, "total": 3 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 4, "total": 4 }, "ineligible_rejection": { "passed": 16, "total": 16 }, "ranking_pair_agreement": { "passed": 3, "total": 3 }, "material_errors": [], "selected_checks": [ { "record_id": "R2814", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-56859", "expected": "ELM-SPO-56859", "passed": true }, { "field": "firm_name", "actual": "Elm Grove 3 Capital", "expected": "Elm Grove 3 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 16560000, "expected": 16560000, "passed": true }, { "field": "event_date", "actual": "2026-04-07", "expected": "2026-04-07", "passed": true }, { "field": "contact_name", "actual": "Jordan Elm3", "expected": "Jordan Elm3", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person3@sponsor-leads-elm.example", "expected": "person3@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R2814" ], "reported_sources": [ "S-R2814" ] }, { "record_id": "R8068", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-69766", "expected": "ELM-SPO-69766", "passed": true }, { "field": "firm_name", "actual": "Elm Square 2 Capital", "expected": "Elm Square 2 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 15480000, "expected": 15480000, "passed": true }, { "field": "event_date", "actual": "2026-03-06", "expected": "2026-03-06", "passed": true }, { "field": "contact_name", "actual": "Drew Elm", "expected": "Drew Elm", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "new.cfo@sponsor-leads-elm.example", "expected": "new.cfo@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R8068", "U-R8068-1" ], "reported_sources": [ "S-R8068", "U-R8068-1" ] }, { "record_id": "R3531", "eligible": true, "duplicate": false, "verified": true, "field_checks": [ { "field": "entity_id", "actual": "ELM-SPO-34913", "expected": "ELM-SPO-34913", "passed": true }, { "field": "firm_name", "actual": "Elm Point 1 Capital", "expected": "Elm Point 1 Capital", "passed": true }, { "field": "transaction_size_usd", "actual": 14400000, "expected": 14400000, "passed": true }, { "field": "event_date", "actual": "2026-02-05", "expected": "2026-02-05", "passed": true }, { "field": "contact_name", "actual": "Alex Elm1", "expected": "Alex Elm1", "passed": true }, { "field": "contact_role", "actual": "CFO", "expected": "CFO", "passed": true }, { "field": "contact_email", "actual": "person1@sponsor-leads-elm.example", "expected": "person1@sponsor-leads-elm.example", "passed": true } ], "evidence_passed": true, "required_sources": [ "S-R3531" ], "reported_sources": [ "S-R3531", "S-R9588" ] } ], "decision_checks": [ { "record_id": "R3531", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R9425", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R4981", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R9588", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4183", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R6876", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3693", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R3654", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R2866", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2827", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R6495", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R4894", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R9706", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R9106", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R2109", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R3292", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R2814", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R4621", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R8068", "actual": true, "expected": true, "passed": true, "reasons": [] }, { "record_id": "R3284", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-fir", "system": "lev/native", "track": "lev_native", "status": "completed", "started_at": "2026-09-07T02:39:05.769Z", "observed_completed_at": "2026-09-07T02:40:14.238Z", "packet_sha256": "98fc64322c0ce87c706cf80ab84a956006eb3d7e47d461d35885d6b2bd118e00", "ui_whitespace_equivalent": true, "operator_semantic_corrections": 0, "artifacts": [ { "path": "artifacts/prospects.csv", "sha256": "bb855c4a769b8446b29a11c6bf8e065139bfdf1582ef082ee0aa30099c6ecd4c", "bytes": 219 } ], "cost_usd": null, "estimated_inference_cost_usd": 0.27594825, "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price", "elapsed_seconds": 62.28, "returned_models": [ "claude-opus-4-7" ], "trace_audit": { "trace_id": "4b0b110b-d638-43ea-aa96-920fec2c8580", "session_id": "e1373412-9252-4ab5-bf7f-ea0dcca26cb5", "started_at": "2026-09-06T22:39:40.944000-04:00", "packet_match": "Trace prefix matches; connector truncates root input. Complete UI input was independently verified before submission.", "event_count": 4, "model_calls": 2, "generation_costs_usd": [ 0.17732675, 0.0986215 ], "total_cost_usd": 0.27594825, "cost_sources": [ "openrouter" ], "models": [ "claude-opus-4-7" ], "agent_versions": [ "7.9" ], "tools_called": [ "get_generated_file" ], "external_retrieval_audit": "no_retrieval_observed_partial_coverage", "execution_coverage": "Recorded backend tools only; internal file-creation execution is not visible in the available trace metadata.", "confirmed_common_corpus": false, "tools_requiring_review": [], "elapsed_seconds": 62.28, "cost_scope": "Matched workflow generations and embeddings; duplicate run-summary excluded. Session naming, hosting, subscriptions, operator time and product credits excluded.", "cost_basis": "PostHog model-rate estimate; not a billed Lev customer price" }, "access_audit": "no_retrieval_observed_partial_coverage", "label": "Lev Agent", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/lev_native/lev--native/sponsor-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-fir", "task": "sponsor_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R4342", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R9787", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1659", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4622", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8409", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9231", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7166", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1915", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R1794", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4732", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9475", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8222", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R6294", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2755", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4936", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R7745", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8586", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4391", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3050", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ] } }, { "case_id": "sponsor-leads-fir", "system": "openai/gpt-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:30:07.582821+00:00", "completed_at": "2026-09-07T03:32:05.442938+00:00", "elapsed_seconds": 117.86, "cost_usd": 0.13906325, "known_partial_cost_usd": 0.13906325, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6684, "completion_tokens": 6522, "total_tokens": 13206, "cost": 0.073575, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 6464, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.073575, "gateway_cost": 0.073575 }, "finish_reason": "tool_calls", "elapsed_seconds": 60.73, "completed_at": "2026-09-07T03:31:08.324513+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 6757, "completion_tokens": 6453, "total_tokens": 13210, "cost": 0.06548825, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 6656, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5248, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.06548825, "gateway_cost": 0.06548825 }, "finish_reason": "tool_calls", "elapsed_seconds": 57.11, "completed_at": "2026-09-07T03:32:05.437907+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "69c3f7317fb834e07ada25f2f3c9b10c92e64b1196e911ce534c47351478cbdc", "bytes": 114 } ], "operator_semantic_corrections": 0, "label": "GPT-5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/openai--gpt-5/sponsor-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-fir", "task": "sponsor_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R4342", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R9787", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1659", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4622", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8409", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9231", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7166", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1915", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R1794", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4732", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9475", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8222", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R6294", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2755", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4936", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R7745", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8586", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4391", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3050", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "69c3f7317fb834e07ada25f2f3c9b10c92e64b1196e911ce534c47351478cbdc" } }, { "case_id": "sponsor-leads-fir", "system": "anthropic/claude-opus-5", "track": "agent", "status": "completed", "started_at": "2026-09-07T03:30:13.689275+00:00", "completed_at": "2026-09-07T03:31:26.415687+00:00", "elapsed_seconds": 72.726, "cost_usd": 0.30383, "known_partial_cost_usd": 0.30383, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 2, "tool_calls": 2, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 10888, "completion_tokens": 1417, "total_tokens": 12305, "cost": 0.089865, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 976, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.089865, "gateway_cost": 0.089865 }, "finish_reason": "tool_calls", "elapsed_seconds": 15.631, "completed_at": "2026-09-07T03:30:29.339660+00:00" }, { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 11358, "completion_tokens": 6287, "total_tokens": 17645, "cost": 0.213965, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 724, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.213965, "gateway_cost": 0.213965 }, "finish_reason": "tool_calls", "elapsed_seconds": 57.07, "completed_at": "2026-09-07T03:31:26.413222+00:00" } ], "artifacts": [ { "path": "artifacts/prospects-v1.csv", "sha256": "30bbfeacc6933713fd61c2e12fbd025cf4265fc5033a961db909e6f62ef5240b", "bytes": 634 } ], "operator_semantic_corrections": 0, "label": "Opus 5 + tools", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/agent/anthropic--claude-opus-5/sponsor-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-fir", "task": "sponsor_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R4342", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R9787", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1659", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4622", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8409", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9231", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7166", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1915", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R1794", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4732", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9475", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8222", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R6294", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2755", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4936", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R7745", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8586", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4391", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3050", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": true, "required_format": ".csv", "checks": [ { "id": "readable_required_format", "passed": true }, { "id": "identity_and_order", "passed": true }, { "id": "selected_facts_and_unknowns", "passed": true }, { "id": "source_identifiers", "passed": true } ], "file": "artifacts/prospects-v1.csv", "criterion": "Original artifact consistency with submitted answer; factual truth is scored separately", "issues": [], "sheets": [ "CSV" ], "table_sheet": "CSV", "original_record_order": [], "answer_record_order": [], "field_checks": [], "harness_induced_null_annotations": 0, "source_checks": [], "source_sha256": "30bbfeacc6933713fd61c2e12fbd025cf4265fc5033a961db909e6f62ef5240b" } }, { "case_id": "sponsor-leads-fir", "system": "openai/gpt-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:31:23.945532+00:00", "completed_at": "2026-09-07T03:32:51.976962+00:00", "elapsed_seconds": 88.031, "cost_usd": 0.0735125, "known_partial_cost_usd": 0.0735125, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "openai/gpt-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "openai/gpt-5", "usage": { "prompt_tokens": 5938, "completion_tokens": 6609, "total_tokens": 12547, "cost": 0.0735125, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 5184, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.0735125, "gateway_cost": 0.0735125 }, "finish_reason": "stop", "elapsed_seconds": 88.013, "completed_at": "2026-09-07T03:32:51.975905+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "GPT-5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/openai--gpt-5/sponsor-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-fir", "task": "sponsor_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R4342", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R9787", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1659", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4622", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8409", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9231", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7166", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1915", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R1794", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4732", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9475", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8222", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R6294", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2755", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4936", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R7745", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8586", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4391", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3050", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } }, { "case_id": "sponsor-leads-fir", "system": "anthropic/claude-opus-5", "track": "direct", "status": "completed", "started_at": "2026-09-07T03:31:26.416593+00:00", "completed_at": "2026-09-07T03:32:11.349417+00:00", "elapsed_seconds": 44.933, "cost_usd": 0.149765, "known_partial_cost_usd": 0.149765, "cost_basis": "Gateway-reported inference including every attempt; excludes operator, hosting and subscription", "returned_models": [ "anthropic/claude-opus-5" ], "model_calls": 1, "tool_calls": 0, "attempts": [ { "attempt": 1, "http_status": 200, "returned_model": "anthropic/claude-opus-5", "usage": { "prompt_tokens": 8603, "completion_tokens": 4270, "total_tokens": 12873, "cost": 0.149765, "is_byok": false, "prompt_tokens_details": { "cached_tokens": 0, "audio_tokens": 0, "video_tokens": 0 }, "cost_details": { "upstream_inference_cost": null, "upstream_inference_prompt_cost": 0, "upstream_inference_completions_cost": 0 }, "completion_tokens_details": { "reasoning_tokens": 1106, "image_tokens": 0 }, "cache_creation_input_tokens": 0, "market_cost": 0.149765, "gateway_cost": 0.149765 }, "finish_reason": "stop", "elapsed_seconds": 44.914, "completed_at": "2026-09-07T03:32:11.347886+00:00" } ], "artifacts": [], "operator_semantic_corrections": 0, "label": "Opus 5 direct", "task": "sponsor_leads", "path": "experiments/2026-09-07-research-v1/direct/anthropic--claude-opus-5/sponsor-leads-fir", "expected_target": 0, "expected_decisions": 20, "grade": { "case_id": "sponsor-leads-fir", "task": "sponsor_leads", "target_count": 0, "available_unique": 0, "returned_count": 0, "eligible_count": 0, "verified_count": 0, "eligible_yield": null, "verified_yield": null, "selection_precision": null, "correct_abstention": true, "field_accuracy": { "passed": 0, "total": 0 }, "evidence_accuracy": { "passed": 0, "total": 0 }, "candidate_accuracy": { "passed": 20, "total": 20 }, "eligible_recall": { "passed": 0, "total": 0 }, "ineligible_rejection": { "passed": 20, "total": 20 }, "ranking_pair_agreement": { "passed": 0, "total": 0 }, "material_errors": [], "selected_checks": [], "decision_checks": [ { "record_id": "R4342", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact is not current" ] }, { "record_id": "R9787", "actual": false, "expected": false, "passed": true, "reasons": [ "Stale contact verification" ] }, { "record_id": "R1659", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R4622", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8409", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9231", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R7166", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R1915", "actual": false, "expected": false, "passed": true, "reasons": [ "Contact role does not qualify" ] }, { "record_id": "R1794", "actual": false, "expected": false, "passed": true, "reasons": [ "Outside transaction size band" ] }, { "record_id": "R4732", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R9475", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R8222", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R6294", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] }, { "record_id": "R2755", "actual": false, "expected": false, "passed": true, "reasons": [ "Event outside lookback" ] }, { "record_id": "R4936", "actual": false, "expected": false, "passed": true, "reasons": [ "Not a closed acquisition" ] }, { "record_id": "R7745", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong asset type" ] }, { "record_id": "R8993", "actual": false, "expected": false, "passed": true, "reasons": [ "Not an evidenced operating sponsor" ] }, { "record_id": "R8586", "actual": false, "expected": false, "passed": true, "reasons": [ "Amount is not a single-asset purchase", "Outside transaction size band" ] }, { "record_id": "R4391", "actual": false, "expected": false, "passed": true, "reasons": [ "Source published after as-of date" ] }, { "record_id": "R3050", "actual": false, "expected": false, "passed": true, "reasons": [ "Wrong market" ] } ], "limits": [ "Classification and shortlist assessment of supplied fictional extracts; not real-world discovery accuracy", "Evidence checks match source IDs and controlling updates; prose rationale requires separate review" ], "scoring_version": "research-v1.1-scoring", "representation_adjustments": [] }, "artifact_audit": { "applicable": false, "reason": "Direct API control has no artifact tools" } } ], "summary": [ { "track": "lev_native", "system": "lev/native", "label": "Lev Agent", "planned": 24, "completed": 24, "graded": 24, "failed": 0, "unanswered_target": 0, "verified": 98, "target": 99, "eligible": 99, "returned": 106, "screening": { "passed": 462, "total": 480, "answered_total": 480, "unanswered": 0 }, "facts": { "passed": 646, "total": 648 }, "evidence": { "passed": 106, "total": 106 }, "ranking": { "passed": 210, "total": 210 }, "abstention": { "passed": 4, "total": 4 }, "material_errors": 9, "cases_with_material_errors": 7, "artifact_checks": { "passed": 86, "total": 96 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 12.61233675, "costs_recorded": 24, "cost_per_verified_usd": 0.12869731377551022 }, { "track": "agent", "system": "openai/gpt-5", "label": "GPT-5 + tools", "planned": 24, "completed": 24, "graded": 24, "failed": 0, "unanswered_target": 0, "verified": 98, "target": 99, "eligible": 99, "returned": 101, "screening": { "passed": 475, "total": 480, "answered_total": 480, "unanswered": 0 }, "facts": { "passed": 615, "total": 615 }, "evidence": { "passed": 100, "total": 101 }, "ranking": { "passed": 210, "total": 210 }, "abstention": { "passed": 4, "total": 4 }, "material_errors": 2, "cases_with_material_errors": 2, "artifact_checks": { "passed": 87, "total": 96 }, "harness_induced_null_annotations": 35, "cases_with_harness_null_loss": 9, "cost_total_usd": 6.34630875, "costs_recorded": 24, "cost_per_verified_usd": 0.06475825255102041 }, { "track": "agent", "system": "anthropic/claude-opus-5", "label": "Opus 5 + tools", "planned": 24, "completed": 24, "graded": 24, "failed": 0, "unanswered_target": 0, "verified": 98, "target": 99, "eligible": 99, "returned": 99, "screening": { "passed": 479, "total": 480, "answered_total": 480, "unanswered": 0 }, "facts": { "passed": 599, "total": 601 }, "evidence": { "passed": 98, "total": 99 }, "ranking": { "passed": 210, "total": 210 }, "abstention": { "passed": 4, "total": 4 }, "material_errors": 2, "cases_with_material_errors": 1, "artifact_checks": { "passed": 88, "total": 96 }, "harness_induced_null_annotations": 33, "cases_with_harness_null_loss": 8, "cost_total_usd": 9.381404999999999, "costs_recorded": 24, "cost_per_verified_usd": 0.09572862244897959 }, { "track": "direct", "system": "openai/gpt-5", "label": "GPT-5 direct", "planned": 24, "completed": 23, "graded": 23, "failed": 1, "unanswered_target": 3, "verified": 94, "target": 99, "eligible": 95, "returned": 98, "screening": { "passed": 455, "total": 480, "answered_total": 460, "unanswered": 20 }, "facts": { "passed": 600, "total": 600 }, "evidence": { "passed": 97, "total": 98 }, "ranking": { "passed": 202, "total": 202 }, "abstention": { "passed": 4, "total": 4 }, "material_errors": 3, "cases_with_material_errors": 3, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 2.31730875, "costs_recorded": 23, "cost_per_verified_usd": null }, { "track": "direct", "system": "anthropic/claude-opus-5", "label": "Opus 5 direct", "planned": 24, "completed": 24, "graded": 24, "failed": 0, "unanswered_target": 0, "verified": 98, "target": 99, "eligible": 99, "returned": 99, "screening": { "passed": 480, "total": 480, "answered_total": 480, "unanswered": 0 }, "facts": { "passed": 601, "total": 601 }, "evidence": { "passed": 98, "total": 99 }, "ranking": { "passed": 210, "total": 210 }, "abstention": { "passed": 4, "total": 4 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 4.49315, "costs_recorded": 24, "cost_per_verified_usd": 0.0458484693877551 } ], "tasks": [ { "track": "lev_native", "system": "lev/native", "label": "Lev Agent", "task": "sales_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 116, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 24, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 3.59528025, "costs_recorded": 6, "cost_per_verified_usd": 0.15631653260869566 }, { "track": "lev_native", "system": "lev/native", "label": "Lev Agent", "task": "rent_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 24, "screening": { "passed": 114, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 120, "total": 120 }, "evidence": { "passed": 24, "total": 24 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 1, "cases_with_material_errors": 1, "artifact_checks": { "passed": 24, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 3.446997, "costs_recorded": 6, "cost_per_verified_usd": 0.1498694347826087 }, { "track": "lev_native", "system": "lev/native", "label": "Lev Agent", "task": "sponsor_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 27, "eligible": 27, "returned": 30, "screening": { "passed": 116, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 208, "total": 210 }, "evidence": { "passed": 30, "total": 30 }, "ranking": { "passed": 63, "total": 63 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 5, "cases_with_material_errors": 3, "artifact_checks": { "passed": 19, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 2.62831775, "costs_recorded": 6, "cost_per_verified_usd": 0.10108914423076923 }, { "track": "lev_native", "system": "lev/native", "label": "Lev Agent", "task": "refinance_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 26, "eligible": 26, "returned": 29, "screening": { "passed": 116, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 203, "total": 203 }, "evidence": { "passed": 29, "total": 29 }, "ranking": { "passed": 61, "total": 61 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 3, "cases_with_material_errors": 3, "artifact_checks": { "passed": 19, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 2.94174175, "costs_recorded": 6, "cost_per_verified_usd": 0.11314391346153846 }, { "track": "agent", "system": "openai/gpt-5", "label": "GPT-5 + tools", "task": "sales_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 24, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 1.14439775, "costs_recorded": 6, "cost_per_verified_usd": 0.049756423913043475 }, { "track": "agent", "system": "openai/gpt-5", "label": "GPT-5 + tools", "task": "rent_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 117, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 24, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 1.9077252500000002, "costs_recorded": 6, "cost_per_verified_usd": 0.08294457608695653 }, { "track": "agent", "system": "openai/gpt-5", "label": "GPT-5 + tools", "task": "sponsor_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 27, "eligible": 27, "returned": 28, "screening": { "passed": 119, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 196, "total": 196 }, "evidence": { "passed": 27, "total": 28 }, "ranking": { "passed": 63, "total": 63 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 1, "cases_with_material_errors": 1, "artifact_checks": { "passed": 20, "total": 24 }, "harness_induced_null_annotations": 4, "cases_with_harness_null_loss": 4, "cost_total_usd": 1.43499775, "costs_recorded": 6, "cost_per_verified_usd": 0.05519222115384615 }, { "track": "agent", "system": "openai/gpt-5", "label": "GPT-5 + tools", "task": "refinance_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 26, "eligible": 26, "returned": 27, "screening": { "passed": 119, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 189, "total": 189 }, "evidence": { "passed": 27, "total": 27 }, "ranking": { "passed": 61, "total": 61 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 1, "cases_with_material_errors": 1, "artifact_checks": { "passed": 19, "total": 24 }, "harness_induced_null_annotations": 31, "cases_with_harness_null_loss": 5, "cost_total_usd": 1.859188, "costs_recorded": 6, "cost_per_verified_usd": 0.07150723076923077 }, { "track": "agent", "system": "anthropic/claude-opus-5", "label": "Opus 5 + tools", "task": "sales_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 24, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 2.26878, "costs_recorded": 6, "cost_per_verified_usd": 0.09864260869565218 }, { "track": "agent", "system": "anthropic/claude-opus-5", "label": "Opus 5 + tools", "task": "rent_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 119, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 24, "total": 24 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 3.56367, "costs_recorded": 6, "cost_per_verified_usd": 0.1549421739130435 }, { "track": "agent", "system": "anthropic/claude-opus-5", "label": "Opus 5 + tools", "task": "sponsor_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 27, "eligible": 27, "returned": 27, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 187, "total": 189 }, "evidence": { "passed": 26, "total": 27 }, "ranking": { "passed": 63, "total": 63 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 2, "cases_with_material_errors": 1, "artifact_checks": { "passed": 21, "total": 24 }, "harness_induced_null_annotations": 3, "cases_with_harness_null_loss": 3, "cost_total_usd": 1.838225, "costs_recorded": 6, "cost_per_verified_usd": 0.07070096153846153 }, { "track": "agent", "system": "anthropic/claude-opus-5", "label": "Opus 5 + tools", "task": "refinance_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 26, "eligible": 26, "returned": 26, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 182, "total": 182 }, "evidence": { "passed": 26, "total": 26 }, "ranking": { "passed": 61, "total": 61 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 19, "total": 24 }, "harness_induced_null_annotations": 30, "cases_with_harness_null_loss": 5, "cost_total_usd": 1.7107299999999999, "costs_recorded": 6, "cost_per_verified_usd": 0.06579730769230768 }, { "track": "direct", "system": "openai/gpt-5", "label": "GPT-5 direct", "task": "sales_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 0.53386875, "costs_recorded": 6, "cost_per_verified_usd": 0.023211684782608696 }, { "track": "direct", "system": "openai/gpt-5", "label": "GPT-5 direct", "task": "rent_comps", "planned": 6, "completed": 5, "graded": 5, "failed": 1, "unanswered_target": 3, "verified": 20, "target": 23, "eligible": 20, "returned": 20, "screening": { "passed": 100, "total": 120, "answered_total": 100, "unanswered": 20 }, "facts": { "passed": 100, "total": 100 }, "evidence": { "passed": 20, "total": 20 }, "ranking": { "passed": 40, "total": 40 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 0.64365875, "costs_recorded": 5, "cost_per_verified_usd": null }, { "track": "direct", "system": "openai/gpt-5", "label": "GPT-5 direct", "task": "sponsor_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 26, "target": 27, "eligible": 27, "returned": 28, "screening": { "passed": 118, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 196, "total": 196 }, "evidence": { "passed": 27, "total": 28 }, "ranking": { "passed": 63, "total": 63 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 1, "cases_with_material_errors": 1, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 0.59084625, "costs_recorded": 6, "cost_per_verified_usd": 0.02272485576923077 }, { "track": "direct", "system": "openai/gpt-5", "label": "GPT-5 direct", "task": "refinance_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 25, "target": 26, "eligible": 25, "returned": 27, "screening": { "passed": 117, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 189, "total": 189 }, "evidence": { "passed": 27, "total": 27 }, "ranking": { "passed": 56, "total": 56 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 2, "cases_with_material_errors": 2, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 0.548935, "costs_recorded": 6, "cost_per_verified_usd": 0.0219574 }, { "track": "direct", "system": "anthropic/claude-opus-5", "label": "Opus 5 direct", "task": "sales_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 0.939415, "costs_recorded": 6, "cost_per_verified_usd": 0.04084413043478261 }, { "track": "direct", "system": "anthropic/claude-opus-5", "label": "Opus 5 direct", "task": "rent_comps", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 23, "target": 23, "eligible": 23, "returned": 23, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 115, "total": 115 }, "evidence": { "passed": 23, "total": 23 }, "ranking": { "passed": 43, "total": 43 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 1.21556, "costs_recorded": 6, "cost_per_verified_usd": 0.052850434782608695 }, { "track": "direct", "system": "anthropic/claude-opus-5", "label": "Opus 5 direct", "task": "sponsor_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 27, "target": 27, "eligible": 27, "returned": 27, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 189, "total": 189 }, "evidence": { "passed": 27, "total": 27 }, "ranking": { "passed": 63, "total": 63 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 1.1696849999999999, "costs_recorded": 6, "cost_per_verified_usd": 0.04332166666666666 }, { "track": "direct", "system": "anthropic/claude-opus-5", "label": "Opus 5 direct", "task": "refinance_leads", "planned": 6, "completed": 6, "graded": 6, "failed": 0, "unanswered_target": 0, "verified": 25, "target": 26, "eligible": 26, "returned": 26, "screening": { "passed": 120, "total": 120, "answered_total": 120, "unanswered": 0 }, "facts": { "passed": 182, "total": 182 }, "evidence": { "passed": 25, "total": 26 }, "ranking": { "passed": 61, "total": 61 }, "abstention": { "passed": 1, "total": 1 }, "material_errors": 0, "cases_with_material_errors": 0, "artifact_checks": { "passed": 0, "total": 0 }, "harness_induced_null_annotations": 0, "cases_with_harness_null_loss": 0, "cost_total_usd": 1.16849, "costs_recorded": 6, "cost_per_verified_usd": 0.0467396 } ] }