{
  "startedAt": "2026-11-17T09:41:56.342Z",
  "model": "deepseek-flash",
  "endpoint": "https://api.deepseek.com/chat/completions",
  "settings": {
    "model": "deepseek-flash",
    "temperature": 0,
    "max_tokens": 700,
    "thinking": {
      "type": "disabled"
    },
    "stream": false
  },
  "sourceType": "synthetic",
  "revisionMethod": "Editorial manual rewrite; not generated by the production optimizer.",
  "repetitions": 1,
  "limitations": "Six English synthetic cases on one provider. Added contract instructions are evaluated against a prespecified exact-match rubric. This is not a model comparison, product efficacy measurement, or estimate of general success rates. Temperature zero does not guarantee deterministic reruns.",
  "prompts": {
    "json": {
      "original": "Extract order_id, amount, currency and status from the source. Return one JSON object with exactly those keys.",
      "revised": "Extract order_id, amount, currency and status using only the source. Return one raw JSON object with exactly those keys, no prose or fences. order_id, currency and status are strings or null; amount is a number or null. Use null for missing or conflicting values. Do not choose one conflicting record without stated precedence. Do not infer currency, payment or zero amounts. Copy supported values without adding facts."
    },
    "evidence": {
      "original": "Summarize the refund rules as JSON with exactly request_window_days, approval, payout_days and sources. Use source IDs in the sources array.",
      "revised": "Extract only stated refund rules. Return one raw JSON object with exactly request_window_days, approval, payout_days and sources, without prose or fences. Day values must be integers or null. Use null for missing or conflicting rules; do not choose precedence when none is provided. approval is \"manual_review\" only when review is explicitly required, otherwise null. sources is a sorted array of the actual source IDs relevant to these findings, including both sides of a conflict. A request window is not a payout deadline. Do not invent policy exceptions or approval guarantees."
    }
  },
  "cases": [
    {
      "id": "json-complete",
      "topic": "json",
      "source": "[A] Order O-17: amount 19.5 USD; status paid.",
      "expected": {
        "order_id": "O-17",
        "amount": 19.5,
        "currency": "USD",
        "status": "paid"
      }
    },
    {
      "id": "json-missing",
      "topic": "json",
      "source": "[A] Order O-18: status pending. Amount and currency are not provided.",
      "expected": {
        "order_id": "O-18",
        "amount": null,
        "currency": null,
        "status": "pending"
      }
    },
    {
      "id": "json-conflict",
      "topic": "json",
      "source": "[A] Order O-19: amount 10 USD; status paid.\n[B] Order O-19: amount 12 USD; status paid. Both records are current; neither has precedence.",
      "expected": {
        "order_id": "O-19",
        "amount": null,
        "currency": "USD",
        "status": "paid"
      }
    },
    {
      "id": "evidence-complete",
      "topic": "evidence",
      "source": "[P1] Refund requests may be submitted within seven days of payment.\n[P2] Approval requires manual review. No payout time is promised.",
      "expected": {
        "request_window_days": 7,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1",
          "P2"
        ]
      }
    },
    {
      "id": "evidence-missing",
      "topic": "evidence",
      "source": "[P1] Approval requires manual review. The request window and payout time are not provided.",
      "expected": {
        "request_window_days": null,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1"
        ]
      }
    },
    {
      "id": "evidence-conflict",
      "topic": "evidence",
      "source": "[P1] Refund requests may be submitted within seven days.\n[P2] Refund requests may be submitted within fourteen days.\n[P3] Approval requires manual review. No payout time is promised. All clauses are current; precedence is unspecified.",
      "expected": {
        "request_window_days": null,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1",
          "P2",
          "P3"
        ]
      }
    }
  ],
  "results": [
    {
      "caseId": "json-complete",
      "variant": "original",
      "input": "Extract order_id, amount, currency and status from the source. Return one JSON object with exactly those keys.\n\n<source>\n[A] Order O-17: amount 19.5 USD; status paid.\n</source>",
      "expected": {
        "order_id": "O-17",
        "amount": 19.5,
        "currency": "USD",
        "status": "paid"
      },
      "output": "{\"order_id\": \"O-17\", \"amount\": 19.5, \"currency\": \"USD\", \"status\": \"paid\"}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 50,
        "completion_tokens": 29,
        "total_tokens": 79,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 50
      },
      "durationMs": 731,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "json-complete",
      "variant": "revised",
      "input": "Extract order_id, amount, currency and status using only the source. Return one raw JSON object with exactly those keys, no prose or fences. order_id, currency and status are strings or null; amount is a number or null. Use null for missing or conflicting values. Do not choose one conflicting record without stated precedence. Do not infer currency, payment or zero amounts. Copy supported values without adding facts.\n\n<source>\n[A] Order O-17: amount 19.5 USD; status paid.\n</source>",
      "expected": {
        "order_id": "O-17",
        "amount": 19.5,
        "currency": "USD",
        "status": "paid"
      },
      "output": "{\"order_id\":\"O-17\",\"amount\":19.5,\"currency\":\"USD\",\"status\":\"paid\"}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 110,
        "completion_tokens": 22,
        "total_tokens": 132,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 110
      },
      "durationMs": 651,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "json-missing",
      "variant": "original",
      "input": "Extract order_id, amount, currency and status from the source. Return one JSON object with exactly those keys.\n\n<source>\n[A] Order O-18: status pending. Amount and currency are not provided.\n</source>",
      "expected": {
        "order_id": "O-18",
        "amount": null,
        "currency": null,
        "status": "pending"
      },
      "output": "{\"order_id\": \"O-18\", \"amount\": null, \"currency\": null, \"status\": \"pending\"}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 50,
        "completion_tokens": 25,
        "total_tokens": 75,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 50
      },
      "durationMs": 779,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "json-missing",
      "variant": "revised",
      "input": "Extract order_id, amount, currency and status using only the source. Return one raw JSON object with exactly those keys, no prose or fences. order_id, currency and status are strings or null; amount is a number or null. Use null for missing or conflicting values. Do not choose one conflicting record without stated precedence. Do not infer currency, payment or zero amounts. Copy supported values without adding facts.\n\n<source>\n[A] Order O-18: status pending. Amount and currency are not provided.\n</source>",
      "expected": {
        "order_id": "O-18",
        "amount": null,
        "currency": null,
        "status": "pending"
      },
      "output": "{\"order_id\":\"O-18\",\"amount\":null,\"currency\":null,\"status\":\"pending\"}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 110,
        "completion_tokens": 20,
        "total_tokens": 130,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 110
      },
      "durationMs": 504,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "json-conflict",
      "variant": "original",
      "input": "Extract order_id, amount, currency and status from the source. Return one JSON object with exactly those keys.\n\n<source>\n[A] Order O-19: amount 10 USD; status paid.\n[B] Order O-19: amount 12 USD; status paid. Both records are current; neither has precedence.\n</source>",
      "expected": {
        "order_id": "O-19",
        "amount": null,
        "currency": "USD",
        "status": "paid"
      },
      "output": "{\"order_id\": \"O-19\", \"amount\": 10, \"currency\": \"USD\", \"status\": \"paid\"}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 72,
        "completion_tokens": 27,
        "total_tokens": 99,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 72
      },
      "durationMs": 908,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": false,
        "passed": false
      }
    },
    {
      "caseId": "json-conflict",
      "variant": "revised",
      "input": "Extract order_id, amount, currency and status using only the source. Return one raw JSON object with exactly those keys, no prose or fences. order_id, currency and status are strings or null; amount is a number or null. Use null for missing or conflicting values. Do not choose one conflicting record without stated precedence. Do not infer currency, payment or zero amounts. Copy supported values without adding facts.\n\n<source>\n[A] Order O-19: amount 10 USD; status paid.\n[B] Order O-19: amount 12 USD; status paid. Both records are current; neither has precedence.\n</source>",
      "expected": {
        "order_id": "O-19",
        "amount": null,
        "currency": "USD",
        "status": "paid"
      },
      "output": "{\"order_id\":\"O-19\",\"amount\":null,\"currency\":\"USD\",\"status\":\"paid\"}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 132,
        "completion_tokens": 20,
        "total_tokens": 152,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 132
      },
      "durationMs": 799,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "evidence-complete",
      "variant": "original",
      "input": "Summarize the refund rules as JSON with exactly request_window_days, approval, payout_days and sources. Use source IDs in the sources array.\n\n<source>\n[P1] Refund requests may be submitted within seven days of payment.\n[P2] Approval requires manual review. No payout time is promised.\n</source>",
      "expected": {
        "request_window_days": 7,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1",
          "P2"
        ]
      },
      "output": "```json\n{\n  \"request_window_days\": 7,\n  \"approval\": \"manual review required\",\n  \"payout_days\": null,\n  \"sources\": [\"P1\", \"P2\"]\n}\n```",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 71,
        "completion_tokens": 48,
        "total_tokens": 119,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 71
      },
      "durationMs": 721,
      "checks": {
        "rawJson": false,
        "exactFields": false,
        "supportedValues": false,
        "passed": false
      }
    },
    {
      "caseId": "evidence-complete",
      "variant": "revised",
      "input": "Extract only stated refund rules. Return one raw JSON object with exactly request_window_days, approval, payout_days and sources, without prose or fences. Day values must be integers or null. Use null for missing or conflicting rules; do not choose precedence when none is provided. approval is \"manual_review\" only when review is explicitly required, otherwise null. sources is a sorted array of the actual source IDs relevant to these findings, including both sides of a conflict. A request window is not a payout deadline. Do not invent policy exceptions or approval guarantees.\n\n<source>\n[P1] Refund requests may be submitted within seven days of payment.\n[P2] Approval requires manual review. No payout time is promised.\n</source>",
      "expected": {
        "request_window_days": 7,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1",
          "P2"
        ]
      },
      "output": "{\"request_window_days\":7,\"approval\":\"manual_review\",\"payout_days\":null,\"sources\":[\"P1\",\"P2\"]}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 155,
        "completion_tokens": 33,
        "total_tokens": 188,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 155
      },
      "durationMs": 734,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "evidence-missing",
      "variant": "original",
      "input": "Summarize the refund rules as JSON with exactly request_window_days, approval, payout_days and sources. Use source IDs in the sources array.\n\n<source>\n[P1] Approval requires manual review. The request window and payout time are not provided.\n</source>",
      "expected": {
        "request_window_days": null,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1"
        ]
      },
      "output": "```json\n{\n  \"request_window_days\": null,\n  \"approval\": \"manual review\",\n  \"payout_days\": null,\n  \"sources\": [\"P1\"]\n}\n```",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 59,
        "completion_tokens": 42,
        "total_tokens": 101,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 59
      },
      "durationMs": 815,
      "checks": {
        "rawJson": false,
        "exactFields": false,
        "supportedValues": false,
        "passed": false
      }
    },
    {
      "caseId": "evidence-missing",
      "variant": "revised",
      "input": "Extract only stated refund rules. Return one raw JSON object with exactly request_window_days, approval, payout_days and sources, without prose or fences. Day values must be integers or null. Use null for missing or conflicting rules; do not choose precedence when none is provided. approval is \"manual_review\" only when review is explicitly required, otherwise null. sources is a sorted array of the actual source IDs relevant to these findings, including both sides of a conflict. A request window is not a payout deadline. Do not invent policy exceptions or approval guarantees.\n\n<source>\n[P1] Approval requires manual review. The request window and payout time are not provided.\n</source>",
      "expected": {
        "request_window_days": null,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1"
        ]
      },
      "output": "{\"request_window_days\":null,\"approval\":\"manual_review\",\"payout_days\":null,\"sources\":[\"P1\"]}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 143,
        "completion_tokens": 30,
        "total_tokens": 173,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 143
      },
      "durationMs": 1342,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    },
    {
      "caseId": "evidence-conflict",
      "variant": "original",
      "input": "Summarize the refund rules as JSON with exactly request_window_days, approval, payout_days and sources. Use source IDs in the sources array.\n\n<source>\n[P1] Refund requests may be submitted within seven days.\n[P2] Refund requests may be submitted within fourteen days.\n[P3] Approval requires manual review. No payout time is promised. All clauses are current; precedence is unspecified.\n</source>",
      "expected": {
        "request_window_days": null,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1",
          "P2",
          "P3"
        ]
      },
      "output": "```json\n{\n  \"request_window_days\": null,\n  \"approval\": \"manual review\",\n  \"payout_days\": null,\n  \"sources\": [\"P1\", \"P2\", \"P3\"]\n}\n```",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 92,
        "completion_tokens": 50,
        "total_tokens": 142,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 92
      },
      "durationMs": 994,
      "checks": {
        "rawJson": false,
        "exactFields": false,
        "supportedValues": false,
        "passed": false
      }
    },
    {
      "caseId": "evidence-conflict",
      "variant": "revised",
      "input": "Extract only stated refund rules. Return one raw JSON object with exactly request_window_days, approval, payout_days and sources, without prose or fences. Day values must be integers or null. Use null for missing or conflicting rules; do not choose precedence when none is provided. approval is \"manual_review\" only when review is explicitly required, otherwise null. sources is a sorted array of the actual source IDs relevant to these findings, including both sides of a conflict. A request window is not a payout deadline. Do not invent policy exceptions or approval guarantees.\n\n<source>\n[P1] Refund requests may be submitted within seven days.\n[P2] Refund requests may be submitted within fourteen days.\n[P3] Approval requires manual review. No payout time is promised. All clauses are current; precedence is unspecified.\n</source>",
      "expected": {
        "request_window_days": null,
        "approval": "manual_review",
        "payout_days": null,
        "sources": [
          "P1",
          "P2",
          "P3"
        ]
      },
      "output": "{\"request_window_days\":null,\"approval\":\"manual_review\",\"payout_days\":null,\"sources\":[\"P1\",\"P2\",\"P3\"]}",
      "providerModel": "deepseek-flash",
      "finishReason": "stop",
      "usage": {
        "prompt_tokens": 176,
        "completion_tokens": 36,
        "total_tokens": 212,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "prompt_cache_hit_tokens": 0,
        "prompt_cache_miss_tokens": 176
      },
      "durationMs": 595,
      "checks": {
        "rawJson": true,
        "exactFields": true,
        "supportedValues": true,
        "passed": true
      }
    }
  ],
  "finishedAt": "2026-11-17T09:42:05.916Z",
  "totals": {
    "original": {
      "passed": 2,
      "total": 6
    },
    "revised": {
      "passed": 6,
      "total": 6
    }
  }
}
