{ "schemaVersion": "1.0.0", "status": "complete", "runId": "20260920T100629126Z-536fff3c", "startedAt": "2026-09-20T10:06:29.127Z", "completedAt": "2026-09-20T10:07:30.453Z", "hashes": { "datasetSha256": "72b2e3f2054febe52a3685659e031dd5c6d66a7f03d9acbd109fd9d7e70bc905", "rubricSha256": "f1900e6b1d4ee2cd7b965d8137121c3d0a8765164b07d813d638cb98f9e2c996", "modelConfigSha256": "b3210238bb7e96e05bd0990799594c746f49d853ee11ec1f724c60514507388e" }, "models": [ { "id": "jev-1.13", "name": "Jev 1.13", "requestedModel": "typesafe/jev-1.13-20260917", "servedModel": "typesafe/jev-1.13-20260917", "upstreamProvider": "TypeSafe", "total": 50, "correct": 50, "accuracy": 1, "errors": 0, "meanLatencyMs": 480.95398, "medianLatencyMs": 451.211, "p95LatencyMs": 629.001, "reportedCostUsd": 0.0034050240000000017, "inputTokens": 81072, "outputTokens": 3340 }, { "id": "gpt-4o-mini", "name": "GPT-4o-mini", "requestedModel": "openai/gpt-4o-mini-2024-07-18", "servedModel": "openai/gpt-4o-mini-2024-07-18", "upstreamProvider": "OpenAI", "total": 50, "correct": 50, "accuracy": 1, "errors": 0, "meanLatencyMs": 742.10762, "medianLatencyMs": 708.225, "p95LatencyMs": 973.68, "reportedCostUsd": 0.005311199999999997, "inputTokens": 35048, "outputTokens": 90 } ], "cases": [ { "id": "DEC-001", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 474.479, "costUsd": 0.000067914 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 973.68, "costUsd": 0.00010515 } } }, { "id": "DEC-002", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 521.617, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 686.056, "costUsd": 0.00010605 } } }, { "id": "DEC-003", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 478.084, "costUsd": 0.000067914 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 702.817, "costUsd": 0.00010575 } } }, { "id": "DEC-004", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 484.109, "costUsd": 0.000067872 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 752.279, "costUsd": 0.0001062 } } }, { "id": "DEC-005", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 442.345, "costUsd": 0.000067872 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 712.142, "costUsd": 0.0001056 } } }, { "id": "DEC-006", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 381.736, "costUsd": 0.000068166 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 679.551, "costUsd": 0.00010605 } } }, { "id": "DEC-007", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 451.211, "costUsd": 0.000067998 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 784.079, "costUsd": 0.00010605 } } }, { "id": "DEC-008", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 520.685, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 705.822, "costUsd": 0.00010605 } } }, { "id": "DEC-009", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 468.133, "costUsd": 0.000067662 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 727.165, "costUsd": 0.00010425 } } }, { "id": "DEC-010", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 616.516, "costUsd": 0.000067872 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 757.692, "costUsd": 0.000105 } } }, { "id": "DEC-011", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 506.807, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 982.534, "costUsd": 0.0001056 } } }, { "id": "DEC-012", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 417.144, "costUsd": 0.000067998 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 697.84, "costUsd": 0.00010605 } } }, { "id": "DEC-013", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 413.055, "costUsd": 0.000067998 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 1168.482, "costUsd": 0.00010605 } } }, { "id": "DEC-014", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 464.171, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 708.225, "costUsd": 0.0001068 } } }, { "id": "DEC-015", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 390.383, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 696.692, "costUsd": 0.0001062 } } }, { "id": "DEC-016", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 438.944, "costUsd": 0.000068082 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 753.884, "costUsd": 0.00010575 } } }, { "id": "DEC-017", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 479.986, "costUsd": 0.000068166 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 672.842, "costUsd": 0.00010665 } } }, { "id": "DEC-018", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 426.744, "costUsd": 0.000067914 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 854.722, "costUsd": 0.00010575 } } }, { "id": "DEC-019", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 1292.045, "costUsd": 0.00006783 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 913.248, "costUsd": 0.00010485 } } }, { "id": "DEC-020", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 361.909, "costUsd": 0.000067746 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 722.543, "costUsd": 0.00010455 } } }, { "id": "DEC-021", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 532.968, "costUsd": 0.00006825 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 733.827, "costUsd": 0.0001062 } } }, { "id": "DEC-022", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 495.775, "costUsd": 0.000068334 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 735.72, "costUsd": 0.00010725 } } }, { "id": "DEC-023", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 591.993, "costUsd": 0.000068124 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 711.534, "costUsd": 0.0001065 } } }, { "id": "DEC-024", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 388.798, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 803.276, "costUsd": 0.00010665 } } }, { "id": "DEC-025", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 568.64, "costUsd": 0.000068124 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 675.694, "costUsd": 0.00010635 } } }, { "id": "DEC-026", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 494.554, "costUsd": 0.00006825 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 721.522, "costUsd": 0.00010635 } } }, { "id": "DEC-027", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 520.897, "costUsd": 0.000068124 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 703.836, "costUsd": 0.0001065 } } }, { "id": "DEC-028", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 389.094, "costUsd": 0.000068082 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 696.698, "costUsd": 0.00010635 } } }, { "id": "DEC-029", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 625.197, "costUsd": 0.000067914 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 724.067, "costUsd": 0.00010485 } } }, { "id": "DEC-030", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 381.676, "costUsd": 0.000067914 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 860.705, "costUsd": 0.00010515 } } }, { "id": "DEC-031", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 471.793, "costUsd": 0.00006846 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 669.022, "costUsd": 0.0001068 } } }, { "id": "DEC-032", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 399.135, "costUsd": 0.000068292 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 698.726, "costUsd": 0.0001071 } } }, { "id": "DEC-033", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 385.235, "costUsd": 0.000068418 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 671.779, "costUsd": 0.00010695 } } }, { "id": "DEC-034", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 413.859, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 718.542, "costUsd": 0.0001068 } } }, { "id": "DEC-035", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 460.005, "costUsd": 0.000068712 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 698.526, "costUsd": 0.0001092 } } }, { "id": "DEC-036", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 396.651, "costUsd": 0.000068208 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 732.812, "costUsd": 0.0001068 } } }, { "id": "DEC-037", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 644.71, "costUsd": 0.000068166 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 700.741, "costUsd": 0.0001071 } } }, { "id": "DEC-038", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 440.559, "costUsd": 0.000068502 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 680.723, "costUsd": 0.00010785 } } }, { "id": "DEC-039", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 429.322, "costUsd": 0.000067956 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 682.171, "costUsd": 0.0001053 } } }, { "id": "DEC-040", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 399.741, "costUsd": 0.00006783 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 767.444, "costUsd": 0.00010485 } } }, { "id": "DEC-041", "outputs": { "jev-1.13": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 629.001, "costUsd": 0.000068292 }, "gpt-4o-mini": { "label": "technical_support", "status": "ok", "correct": true, "latencyMs": 662.106, "costUsd": 0.0001068 } } }, { "id": "DEC-042", "outputs": { "jev-1.13": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 407.677, "costUsd": 0.00006825 }, "gpt-4o-mini": { "label": "account_issue", "status": "ok", "correct": true, "latencyMs": 694.944, "costUsd": 0.0001068 } } }, { "id": "DEC-043", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 535.191, "costUsd": 0.000068124 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 746.306, "costUsd": 0.00010695 } } }, { "id": "DEC-044", "outputs": { "jev-1.13": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 405.954, "costUsd": 0.000068334 }, "gpt-4o-mini": { "label": "product_inquiry", "status": "ok", "correct": true, "latencyMs": 704.503, "costUsd": 0.0001077 } } }, { "id": "DEC-045", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 399.813, "costUsd": 0.00006825 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 665.246, "costUsd": 0.0001068 } } }, { "id": "DEC-046", "outputs": { "jev-1.13": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 404.761, "costUsd": 0.00006804 }, "gpt-4o-mini": { "label": "refund", "status": "ok", "correct": true, "latencyMs": 656.508, "costUsd": 0.0001056 } } }, { "id": "DEC-047", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 416.802, "costUsd": 0.000068208 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 691.76, "costUsd": 0.00010665 } } }, { "id": "DEC-048", "outputs": { "jev-1.13": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 451.897, "costUsd": 0.000068292 }, "gpt-4o-mini": { "label": "complaint", "status": "ok", "correct": true, "latencyMs": 697.424, "costUsd": 0.00010695 } } }, { "id": "DEC-049", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 428.199, "costUsd": 0.000068082 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 798.111, "costUsd": 0.0001056 } } }, { "id": "DEC-050", "outputs": { "jev-1.13": { "label": "other", "status": "ok", "correct": true, "latencyMs": 507.699, "costUsd": 0.000068208 }, "gpt-4o-mini": { "label": "other", "status": "ok", "correct": true, "latencyMs": 748.813, "costUsd": 0.00010605 } } } ], "probeCostUsd": 0.000174816, "limitations": [ "Exploratory pilot: 50 synthetic, AI-authored and AI-reviewed messages; no human/expert labels or production sample.", "One scored attempt per case and model, with no automatic retries. Each case changes the accuracy by 2 percentage points.", "Both models receive the same semantic rubric through provider-specific API formats. Jev criteria repeat label definitions, so token counts are not matched.", "Latency is client-observed end-to-end time, including network and gateway overhead. Requests are sequential and model order alternates by case.", "Costs are reported inference usage charges for the scored batch; they exclude the separate connectivity probes and account funding fees.", "This sample cannot establish a general model ranking, production reliability, or statistically reliable small differences." ] }