{
  "$id": "https://crow.sg/research/llm-arbitrary-choice-study/study-summary.schema.json",
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "additionalProperties": false,
  "description": "Machine-readable JSON Schema inferred from the generated public study summary artifact.",
  "properties": {
    "attemptsByMaxTokens": {
      "description": "Attempt outcomes grouped by status and max_tokens setting.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "count": {
            "minimum": 0,
            "type": "integer"
          },
          "maxTokens": {
            "minimum": 0,
            "type": "integer"
          },
          "status": {
            "enum": [
              "error",
              "invalid",
              "model_removed",
              "ok",
              "rate_limited"
            ],
            "type": "string"
          }
        },
        "required": [
          "count",
          "maxTokens",
          "status"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "caveats": {
      "description": "Public caveats, operational limits, and AI-assistance disclosure.",
      "items": {
        "type": "string"
      },
      "type": "array"
    },
    "conditionSummary": {
      "description": "Prompt-condition totals, OK rates, and first-displayed-option shares.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "condition": {
            "enum": [
              "bare",
              "bare_swapped",
              "context",
              "context_swapped"
            ],
            "type": "string"
          },
          "firstShare": {
            "description": "Share of OK rows where parsed_choice matched the first displayed option.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "okRate": {
            "description": "Share of planned trial rows with final status ok for the grouping.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "okTrials": {
            "minimum": 0,
            "type": "integer"
          },
          "totalTrials": {
            "minimum": 0,
            "type": "integer"
          }
        },
        "required": [
          "condition",
          "firstShare",
          "okRate",
          "okTrials",
          "totalTrials"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "contextEffects": {
      "description": "Context-prompt lift versus the same model and pair under bare prompts.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "absMeanLift": {
            "description": "Absolute value of meanLift, used for sorting strongest context effects.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "baselineShare": {
            "description": "Share for the intended option under bare prompts for the same model and pair.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "contextId": {
            "type": "string"
          },
          "contextShare": {
            "description": "Share for the intended option under context prompts for the same model and pair.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "intendedAssociation": {
            "type": "string"
          },
          "intendedOption": {
            "type": "string"
          },
          "meanLift": {
            "description": "Mean per-model change in intended-option share under context prompts versus bare baseline.",
            "maximum": 1,
            "minimum": -1,
            "type": "number"
          },
          "models": {
            "minimum": 0,
            "type": "integer"
          },
          "n": {
            "minimum": 0,
            "type": "integer"
          },
          "pairId": {
            "type": "string"
          },
          "text": {
            "type": "string"
          }
        },
        "required": [
          "absMeanLift",
          "baselineShare",
          "contextId",
          "contextShare",
          "intendedAssociation",
          "intendedOption",
          "meanLift",
          "models",
          "n",
          "pairId",
          "text"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "cost": {
      "additionalProperties": false,
      "description": "Dashboard spend, captured usage spend, and token accounting from recorded attempts.",
      "properties": {
        "attemptCompletionTokens": {
          "minimum": 0,
          "type": "integer"
        },
        "attemptPromptTokens": {
          "minimum": 0,
          "type": "integer"
        },
        "attemptReasoningTokens": {
          "minimum": 0,
          "type": "integer"
        },
        "attemptUsageObservedUsd": {
          "minimum": 0,
          "type": "number"
        },
        "dashboardSpendUsd": {
          "description": "Displayed OpenRouter dashboard spend for the study run.",
          "minimum": 0,
          "type": "number"
        },
        "note": {
          "type": "string"
        },
        "trialUsageObservedUsd": {
          "minimum": 0,
          "type": "number"
        }
      },
      "required": [
        "attemptCompletionTokens",
        "attemptPromptTokens",
        "attemptReasoningTokens",
        "attemptUsageObservedUsd",
        "dashboardSpendUsd",
        "note",
        "trialUsageObservedUsd"
      ],
      "type": "object"
    },
    "insights": {
      "additionalProperties": false,
      "description": "Derived headline values and selected examples used by the public article and charts.",
      "properties": {
        "acceptedNonExactOkRows": {
          "minimum": 0,
          "type": "integer"
        },
        "exactParseOkRows": {
          "minimum": 0,
          "type": "integer"
        },
        "exactParseOkShare": {
          "maximum": 1,
          "minimum": 0,
          "type": "number"
        },
        "manualOverrideRows": {
          "minimum": 0,
          "type": "integer"
        },
        "mostFirstOptionModel": {
          "additionalProperties": false,
          "properties": {
            "firstShare": {
              "description": "Share of OK rows where parsed_choice matched the first displayed option.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "label": {
              "type": "string"
            },
            "modelId": {
              "type": "string"
            },
            "provider": {
              "type": "string"
            }
          },
          "required": [
            "firstShare",
            "label",
            "modelId",
            "provider"
          ],
          "type": "object"
        },
        "mostNeutralPair": {
          "additionalProperties": false,
          "properties": {
            "category": {
              "type": "string"
            },
            "majorityOption": {
              "type": "string"
            },
            "n": {
              "minimum": 0,
              "type": "integer"
            },
            "optionA": {
              "type": "string"
            },
            "optionACount": {
              "minimum": 0,
              "type": "integer"
            },
            "optionAShare": {
              "description": "Share of OK bare and swapped rows where parsed_choice matched optionA.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "optionB": {
              "type": "string"
            },
            "optionBShare": {
              "description": "Share of OK bare and swapped rows where parsed_choice matched optionB.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "pairId": {
              "type": "string"
            },
            "preferenceStrength": {
              "description": "Absolute distance from an even option split, scaled from 0 to 1.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            }
          },
          "required": [
            "category",
            "majorityOption",
            "n",
            "optionA",
            "optionACount",
            "optionAShare",
            "optionB",
            "optionBShare",
            "pairId",
            "preferenceStrength"
          ],
          "type": "object"
        },
        "mostSecondOptionModel": {
          "additionalProperties": false,
          "properties": {
            "firstShare": {
              "description": "Share of OK rows where parsed_choice matched the first displayed option.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "label": {
              "type": "string"
            },
            "modelId": {
              "type": "string"
            },
            "provider": {
              "type": "string"
            }
          },
          "required": [
            "firstShare",
            "label",
            "modelId",
            "provider"
          ],
          "type": "object"
        },
        "overallFirstShare": {
          "maximum": 1,
          "minimum": 0,
          "type": "number"
        },
        "rekaFlashTokenShare": {
          "additionalProperties": false,
          "properties": {
            "attemptCostShare": {
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "completionTokenShare": {
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "okRows": {
              "minimum": 0,
              "type": "integer"
            },
            "reasoningTokenShare": {
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            }
          },
          "required": [
            "attemptCostShare",
            "completionTokenShare",
            "okRows",
            "reasoningTokenShare"
          ],
          "type": "object"
        },
        "strongestPositiveContext": {
          "additionalProperties": false,
          "properties": {
            "absMeanLift": {
              "description": "Absolute value of meanLift, used for sorting strongest context effects.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "baselineShare": {
              "description": "Share for the intended option under bare prompts for the same model and pair.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "contextId": {
              "type": "string"
            },
            "contextShare": {
              "description": "Share for the intended option under context prompts for the same model and pair.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "intendedAssociation": {
              "type": "string"
            },
            "intendedOption": {
              "type": "string"
            },
            "meanLift": {
              "description": "Mean per-model change in intended-option share under context prompts versus bare baseline.",
              "maximum": 1,
              "minimum": -1,
              "type": "number"
            },
            "models": {
              "minimum": 0,
              "type": "integer"
            },
            "n": {
              "minimum": 0,
              "type": "integer"
            },
            "pairId": {
              "type": "string"
            },
            "text": {
              "type": "string"
            }
          },
          "required": [
            "absMeanLift",
            "baselineShare",
            "contextId",
            "contextShare",
            "intendedAssociation",
            "intendedOption",
            "meanLift",
            "models",
            "n",
            "pairId",
            "text"
          ],
          "type": "object"
        },
        "strongestReverseContext": {
          "additionalProperties": false,
          "properties": {
            "absMeanLift": {
              "description": "Absolute value of meanLift, used for sorting strongest context effects.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "baselineShare": {
              "description": "Share for the intended option under bare prompts for the same model and pair.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "contextId": {
              "type": "string"
            },
            "contextShare": {
              "description": "Share for the intended option under context prompts for the same model and pair.",
              "maximum": 1,
              "minimum": 0,
              "type": "number"
            },
            "intendedAssociation": {
              "type": "string"
            },
            "intendedOption": {
              "type": "string"
            },
            "meanLift": {
              "description": "Mean per-model change in intended-option share under context prompts versus bare baseline.",
              "maximum": 1,
              "minimum": -1,
              "type": "number"
            },
            "models": {
              "minimum": 0,
              "type": "integer"
            },
            "n": {
              "minimum": 0,
              "type": "integer"
            },
            "pairId": {
              "type": "string"
            },
            "text": {
              "type": "string"
            }
          },
          "required": [
            "absMeanLift",
            "baselineShare",
            "contextId",
            "contextShare",
            "intendedAssociation",
            "intendedOption",
            "meanLift",
            "models",
            "n",
            "pairId",
            "text"
          ],
          "type": "object"
        },
        "topNonOkModels": {
          "items": {
            "additionalProperties": false,
            "properties": {
              "label": {
                "type": "string"
              },
              "modelId": {
                "type": "string"
              },
              "nonOkRows": {
                "minimum": 0,
                "type": "integer"
              },
              "okRows": {
                "minimum": 0,
                "type": "integer"
              },
              "provider": {
                "type": "string"
              },
              "shareOfAllNonOk": {
                "maximum": 1,
                "minimum": 0,
                "type": "number"
              },
              "status": {
                "additionalProperties": false,
                "properties": {
                  "error": {
                    "minimum": 0,
                    "type": "integer"
                  },
                  "invalid": {
                    "minimum": 0,
                    "type": "integer"
                  },
                  "model_removed": {
                    "minimum": 0,
                    "type": "integer"
                  },
                  "ok": {
                    "minimum": 0,
                    "type": "integer"
                  },
                  "rate_limited": {
                    "minimum": 0,
                    "type": "integer"
                  }
                },
                "type": "object"
              },
              "totalTrials": {
                "minimum": 0,
                "type": "integer"
              }
            },
            "required": [
              "label",
              "modelId",
              "nonOkRows",
              "okRows",
              "provider",
              "shareOfAllNonOk",
              "status",
              "totalTrials"
            ],
            "type": "object"
          },
          "type": "array"
        },
        "topTwoNonOkShare": {
          "maximum": 1,
          "minimum": 0,
          "type": "number"
        },
        "totalNonOkRows": {
          "minimum": 0,
          "type": "integer"
        }
      },
      "required": [
        "acceptedNonExactOkRows",
        "exactParseOkRows",
        "exactParseOkShare",
        "manualOverrideRows",
        "mostFirstOptionModel",
        "mostNeutralPair",
        "mostSecondOptionModel",
        "overallFirstShare",
        "rekaFlashTokenShare",
        "strongestPositiveContext",
        "strongestReverseContext",
        "topNonOkModels",
        "topTwoNonOkShare",
        "totalNonOkRows"
      ],
      "type": "object"
    },
    "modelPairBias": {
      "description": "Per-model option preference for each word pair over bare and swapped prompts.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "category": {
            "type": "string"
          },
          "firstShare": {
            "description": "Share of OK rows where parsed_choice matched the first displayed option.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "majorityOption": {
            "type": "string"
          },
          "modelId": {
            "type": "string"
          },
          "n": {
            "minimum": 0,
            "type": "integer"
          },
          "optionA": {
            "type": "string"
          },
          "optionAShare": {
            "description": "Share of OK bare and swapped rows where parsed_choice matched optionA.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "optionB": {
            "type": "string"
          },
          "pairId": {
            "type": "string"
          },
          "preferenceStrength": {
            "description": "Absolute distance from an even option split, scaled from 0 to 1.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          }
        },
        "required": [
          "category",
          "firstShare",
          "majorityOption",
          "modelId",
          "n",
          "optionA",
          "optionAShare",
          "optionB",
          "pairId",
          "preferenceStrength"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "models": {
      "description": "Per-model reliability, bias, context-sensitivity, taxonomy, and captured usage summary.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "attemptCostUsd": {
            "description": "Captured OpenRouter usage cost from recorded attempt rows for the grouping.",
            "minimum": 0,
            "type": "number"
          },
          "bareFirstShare": {
            "maximum": 1,
            "minimum": 0,
            "type": [
              "null",
              "number"
            ]
          },
          "completionTokens": {
            "minimum": 0,
            "type": "integer"
          },
          "contextSensitivity": {
            "description": "Mean absolute context lift for a model over context rows with data.",
            "maximum": 1,
            "minimum": 0,
            "type": [
              "null",
              "number"
            ]
          },
          "family": {
            "type": "string"
          },
          "firstShare": {
            "description": "Share of OK rows where parsed_choice matched the first displayed option.",
            "maximum": 1,
            "minimum": 0,
            "type": [
              "null",
              "number"
            ]
          },
          "label": {
            "type": "string"
          },
          "modelId": {
            "type": "string"
          },
          "okRate": {
            "description": "Share of planned trial rows with final status ok for the grouping.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "okTrials": {
            "minimum": 0,
            "type": "integer"
          },
          "origin": {
            "enum": [
              "China",
              "EU",
              "Other",
              "US",
              "Unknown"
            ],
            "type": "string"
          },
          "positionBiasStrength": {
            "description": "Absolute distance from a 50 percent first-option share, scaled from 0 to 1.",
            "maximum": 1,
            "minimum": 0,
            "type": [
              "null",
              "number"
            ]
          },
          "promptTokens": {
            "minimum": 0,
            "type": "integer"
          },
          "provider": {
            "type": "string"
          },
          "reasoningTokens": {
            "minimum": 0,
            "type": "integer"
          },
          "researchClass": {
            "enum": [
              "current mid-tier",
              "flagship / frontier-class",
              "open-weight or open-route",
              "reasoning-specialized",
              "small or fast tier",
              "uncategorized"
            ],
            "type": "string"
          },
          "semanticPreferenceStrength": {
            "description": "Mean word-pair preference strength for a model over pairs with enough bare and swapped data.",
            "maximum": 1,
            "minimum": 0,
            "type": [
              "null",
              "number"
            ]
          },
          "status": {
            "additionalProperties": false,
            "properties": {
              "error": {
                "minimum": 0,
                "type": "integer"
              },
              "invalid": {
                "minimum": 0,
                "type": "integer"
              },
              "model_removed": {
                "minimum": 0,
                "type": "integer"
              },
              "ok": {
                "minimum": 0,
                "type": "integer"
              },
              "rate_limited": {
                "minimum": 0,
                "type": "integer"
              }
            },
            "type": "object"
          },
          "tier": {
            "enum": [
              "flagship",
              "mid",
              "open",
              "reasoning",
              "small",
              "unknown"
            ],
            "type": "string"
          },
          "totalTrials": {
            "minimum": 0,
            "type": "integer"
          },
          "usageRows": {
            "minimum": 0,
            "type": "integer"
          }
        },
        "required": [
          "attemptCostUsd",
          "bareFirstShare",
          "completionTokens",
          "contextSensitivity",
          "family",
          "firstShare",
          "label",
          "modelId",
          "okRate",
          "okTrials",
          "origin",
          "positionBiasStrength",
          "promptTokens",
          "provider",
          "reasoningTokens",
          "researchClass",
          "semanticPreferenceStrength",
          "status",
          "tier",
          "totalTrials",
          "usageRows"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "pairBias": {
      "description": "Aggregate option preference for each word pair over bare and swapped prompts.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "category": {
            "type": "string"
          },
          "majorityOption": {
            "type": "string"
          },
          "n": {
            "minimum": 0,
            "type": "integer"
          },
          "optionA": {
            "type": "string"
          },
          "optionACount": {
            "minimum": 0,
            "type": "integer"
          },
          "optionAShare": {
            "description": "Share of OK bare and swapped rows where parsed_choice matched optionA.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "optionB": {
            "type": "string"
          },
          "optionBShare": {
            "description": "Share of OK bare and swapped rows where parsed_choice matched optionB.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "pairId": {
            "type": "string"
          },
          "preferenceStrength": {
            "description": "Absolute distance from an even option split, scaled from 0 to 1.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          }
        },
        "required": [
          "category",
          "majorityOption",
          "n",
          "optionA",
          "optionACount",
          "optionAShare",
          "optionB",
          "optionBShare",
          "pairId",
          "preferenceStrength"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "parseStatusCounts": {
      "description": "Parser decision totals across all trial rows.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "count": {
            "minimum": 0,
            "type": "integer"
          },
          "parseStatus": {
            "enum": [
              "exact",
              "invalid",
              "manual_override",
              "none",
              "repeated_single_option",
              "single_token_in_text"
            ],
            "type": "string"
          }
        },
        "required": [
          "count",
          "parseStatus"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "providerSummary": {
      "description": "Per-provider rollup of trial status counts, OK rate, model count, spend, and mean bias metrics.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "attemptCostUsd": {
            "description": "Captured OpenRouter usage cost from recorded attempt rows for the grouping.",
            "minimum": 0,
            "type": "number"
          },
          "counts": {
            "additionalProperties": false,
            "properties": {
              "error": {
                "minimum": 0,
                "type": "integer"
              },
              "invalid": {
                "minimum": 0,
                "type": "integer"
              },
              "model_removed": {
                "minimum": 0,
                "type": "integer"
              },
              "ok": {
                "minimum": 0,
                "type": "integer"
              },
              "rate_limited": {
                "minimum": 0,
                "type": "integer"
              }
            },
            "required": [
              "ok"
            ],
            "type": "object"
          },
          "meanFirstShare": {
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "meanSemanticPreferenceStrength": {
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "models": {
            "minimum": 0,
            "type": "integer"
          },
          "okRate": {
            "description": "Share of planned trial rows with final status ok for the grouping.",
            "maximum": 1,
            "minimum": 0,
            "type": "number"
          },
          "provider": {
            "type": "string"
          },
          "totalTrials": {
            "minimum": 0,
            "type": "integer"
          }
        },
        "required": [
          "attemptCostUsd",
          "counts",
          "meanFirstShare",
          "meanSemanticPreferenceStrength",
          "models",
          "okRate",
          "provider",
          "totalTrials"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "statusCounts": {
      "description": "Trial rows grouped by final execution or parse status.",
      "items": {
        "additionalProperties": false,
        "properties": {
          "count": {
            "minimum": 0,
            "type": "integer"
          },
          "status": {
            "enum": [
              "error",
              "invalid",
              "model_removed",
              "ok",
              "rate_limited"
            ],
            "type": "string"
          }
        },
        "required": [
          "count",
          "status"
        ],
        "type": "object"
      },
      "type": "array"
    },
    "study": {
      "additionalProperties": false,
      "description": "Study metadata, source timestamps, run configuration, and public headline counts.",
      "properties": {
        "articleDate": {
          "pattern": "^\\d{4}-\\d{2}-\\d{2}$",
          "type": "string"
        },
        "attemptsUpdatedAt": {
          "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$",
          "type": "string"
        },
        "author": {
          "type": "string"
        },
        "conditions": {
          "items": {
            "enum": [
              "bare",
              "bare_swapped",
              "context",
              "context_swapped"
            ],
            "type": "string"
          },
          "type": "array"
        },
        "contexts": {
          "minimum": 0,
          "type": "integer"
        },
        "dashboardSpendUsd": {
          "description": "Displayed OpenRouter dashboard spend for the study run.",
          "minimum": 0,
          "type": "number"
        },
        "inferredContextTargets": {
          "minimum": 0,
          "type": "integer"
        },
        "models": {
          "minimum": 0,
          "type": "integer"
        },
        "pairs": {
          "minimum": 0,
          "type": "integer"
        },
        "plannedTrials": {
          "minimum": 0,
          "type": "integer"
        },
        "repetitions": {
          "minimum": 0,
          "type": "integer"
        },
        "slug": {
          "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
          "type": "string"
        },
        "sourceDatabase": {
          "type": "string"
        },
        "sourceUpdatedAt": {
          "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$",
          "type": "string"
        },
        "subtitle": {
          "type": "string"
        },
        "taxonomyNote": {
          "type": "string"
        },
        "temperature": {
          "minimum": 0,
          "type": "number"
        },
        "title": {
          "type": "string"
        }
      },
      "required": [
        "articleDate",
        "attemptsUpdatedAt",
        "author",
        "conditions",
        "contexts",
        "dashboardSpendUsd",
        "inferredContextTargets",
        "models",
        "pairs",
        "plannedTrials",
        "repetitions",
        "slug",
        "sourceDatabase",
        "sourceUpdatedAt",
        "subtitle",
        "taxonomyNote",
        "temperature",
        "title"
      ],
      "type": "object"
    }
  },
  "required": [
    "attemptsByMaxTokens",
    "caveats",
    "conditionSummary",
    "contextEffects",
    "cost",
    "insights",
    "modelPairBias",
    "models",
    "pairBias",
    "parseStatusCounts",
    "providerSummary",
    "statusCounts",
    "study"
  ],
  "title": "Arbitrary Choices Are Not Random public summary schema",
  "type": "object"
}