{
  "title": "When AI Agents Design the Test",
  "date": "2026-10-03",
  "new_model_calls": 0,
  "scope": "Frozen-response analyses. Target-specific available reference intersections; no paired repair experiment or repeated research episodes.",
  "sources": [
    {
      "file": "astra-truncation-sensitivity.json",
      "sha256": "5f0263feef3db01e23f55e50f81952e475979db4ffc93ee54bdb72b9c84dfb6d"
    },
    {
      "file": "ccc-frozen-component-subsets.json",
      "sha256": "81c0deb1d52ffb4011cdbec2485d50ac2f6b653b81e265f67555054995c097ca"
    },
    {
      "file": "grok-weight-counterfactual.json",
      "sha256": "61edc110fb76ee6e2a78e2efaa9245daf499f56312e2851a0357c2c1abad2e6b"
    },
    {
      "file": "data.json",
      "sha256": "04831a8bf39be9369eb87c7f7db8a9a80f293f151594ed3229beda9313bcb7f5"
    },
    {
      "file": "benchmark.json",
      "sha256": "669523e2f81f8e4fb7bc71d5a64844412a1f9f39ceb2e487b6a7ba8c2f8089eb"
    },
    {
      "file": "figure-data.json",
      "sha256": "97db5a1a3d3107697fe73c67ae0728675cc07d483c33f2f78c473b37fbb85c75"
    }
  ],
  "board": [
    {
      "n": "GPT-6 Astra",
      "key": "astra",
      "d": {
        "dev": [
          [
            -5.19,
            40.41,
            60.64,
            20.98
          ],
          [
            55.71,
            75.76,
            83.33,
            66.99
          ]
        ],
        "sealed": [
          [
            4.79,
            43.16,
            24.69,
            24.15
          ],
          [
            60.32,
            79.47,
            76.09,
            71.44
          ]
        ],
        "all": [
          [
            -0.2,
            42.06,
            36.67,
            22.79
          ],
          [
            58.01,
            77.98,
            78.51,
            69.54
          ]
        ]
      }
    },
    {
      "n": "Grok 4.7",
      "key": "grok",
      "d": {
        "dev": [
          [
            24.35,
            10.34,
            13.33,
            17.84
          ],
          [
            77.49,
            72.1,
            82.22,
            76.48
          ]
        ],
        "sealed": [
          [
            22.22,
            37.84,
            14.48,
            26.14
          ],
          [
            66.85,
            81.63,
            71.93,
            73.66
          ]
        ],
        "all": [
          [
            23.28,
            26.84,
            14.1,
            22.59
          ],
          [
            72.17,
            77.81,
            75.36,
            74.87
          ]
        ]
      }
    },
    {
      "n": "Fable 5.1",
      "key": "fable",
      "d": {
        "dev": [
          [
            9.81,
            23.52,
            63.94,
            23.4
          ],
          [
            61.03,
            69.87,
            81.11,
            67.32
          ]
        ],
        "sealed": [
          [
            20.92,
            19.68,
            22.6,
            20.87
          ],
          [
            63.94,
            73.13,
            67.05,
            68.16
          ]
        ],
        "all": [
          [
            15.36,
            21.22,
            36.38,
            21.96
          ],
          [
            62.48,
            71.82,
            71.73,
            67.8
          ]
        ]
      }
    },
    {
      "n": "Opus 5",
      "key": "opus",
      "d": {
        "dev": [
          [
            -6.36,
            24.34,
            77.99,
            17.93
          ],
          [
            51.26,
            69.02,
            80.0,
            61.97
          ]
        ],
        "sealed": [
          [
            5.75,
            47.6,
            18.7,
            24.68
          ],
          [
            56.3,
            74.61,
            71.89,
            67.06
          ]
        ],
        "all": [
          [
            -0.31,
            38.3,
            38.46,
            21.79
          ],
          [
            53.78,
            72.37,
            74.59,
            64.88
          ]
        ]
      }
    },
    {
      "n": "Kimi K3",
      "key": "k3",
      "d": {
        "dev": [
          [
            6.49,
            12.34,
            55.33,
            16.58
          ],
          [
            64.33,
            74.07,
            81.11,
            70.38
          ]
        ],
        "sealed": [
          [
            7.93,
            24.16,
            28.35,
            19.12
          ],
          [
            70.88,
            87.07,
            73.99,
            77.73
          ]
        ],
        "all": [
          [
            7.21,
            19.43,
            37.35,
            18.03
          ],
          [
            67.61,
            81.87,
            76.36,
            74.58
          ]
        ]
      }
    },
    {
      "n": "GPT-5.6 Sol",
      "key": "sol",
      "d": {
        "dev": [
          [
            21.9,
            -6.66,
            50.45,
            17.14
          ],
          [
            66.94,
            46.38,
            81.11,
            62.45
          ]
        ],
        "sealed": [
          [
            17.71,
            10.24,
            25.58,
            16.88
          ],
          [
            60.6,
            60.31,
            74.54,
            63.98
          ]
        ],
        "all": [
          [
            19.81,
            3.48,
            33.87,
            16.99
          ],
          [
            63.77,
            54.74,
            76.73,
            63.32
          ]
        ]
      }
    },
    {
      "n": "GLM-5.3",
      "key": "glm",
      "d": {
        "dev": [
          [
            1.86,
            11.68,
            50.0,
            13.15
          ],
          [
            56.67,
            67.51,
            72.22,
            62.87
          ]
        ],
        "sealed": [
          [
            8.73,
            18.3,
            19.91,
            15.12
          ],
          [
            53.24,
            77.57,
            68.18,
            66.1
          ]
        ],
        "all": [
          [
            5.29,
            15.65,
            29.94,
            14.27
          ],
          [
            54.95,
            73.55,
            69.53,
            64.72
          ]
        ]
      }
    },
    {
      "n": "DeepSeek V4 Pro",
      "key": "ds",
      "d": {
        "dev": [
          [
            3.2,
            5.55,
            44.0,
            10.78
          ],
          [
            56.78,
            64.27,
            74.44,
            62.22
          ]
        ],
        "sealed": [
          [
            -0.53,
            11.59,
            24.51,
            10.28
          ],
          [
            63.19,
            65.91,
            64.18,
            64.46
          ]
        ],
        "all": [
          [
            1.33,
            9.17,
            31.01,
            10.49
          ],
          [
            59.99,
            65.25,
            67.6,
            63.5
          ]
        ]
      }
    },
    {
      "n": "Random120",
      "key": "r120",
      "base": true,
      "d": {
        "dev": [
          [
            21.44,
            -10.71,
            12.79,
            9.28
          ],
          [
            73.87,
            55.96,
            68.06,
            66.93
          ]
        ],
        "sealed": [
          [
            24.39,
            13.39,
            7.15,
            15.96
          ],
          [
            76.63,
            62.06,
            69.2,
            69.31
          ]
        ],
        "all": [
          [
            22.92,
            3.75,
            9.03,
            13.1
          ],
          [
            75.25,
            59.62,
            68.82,
            68.29
          ]
        ]
      }
    }
  ],
  "grok_weights": {
    "coding": [
      0.034,
      0.04,
      0.049,
      0.049,
      0.049,
      0.049,
      0.049,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.034,
      0.034,
      0.00015,
      0.049,
      0.049,
      0.049,
      0.049,
      0.049,
      0.00015,
      0.049,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.049,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.049,
      0.00015,
      0.00015,
      0.00015,
      0.034,
      0.034,
      0.049,
      0.00015,
      0.049,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.049,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015,
      0.00015
    ],
    "co-work": [
      0.011,
      0.0125,
      0.15645,
      0.15645,
      0.15645,
      0.15645,
      0.15645,
      0.0002,
      0.0125,
      0.0002,
      0.011,
      0.15645,
      0.0125,
      0.0002,
      0.0002,
      0.0002,
      0.0002,
      0.0002,
      0.0002,
      0.0002
    ],
    "reasoning": [
      0.073,
      0.016,
      0.016,
      0.004,
      0.05234117,
      0.05234117,
      0.05234117,
      0.05234117,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.05234118,
      0.0003,
      0.0003,
      0.05234117,
      0.0003,
      0.0003,
      0.05234117
    ]
  },
  "grok_reweight": {
    "m12": [
      [
        "Coding",
        22.94,
        73.9,
        36.85,
        73.83
      ],
      [
        "Co-work",
        25.75,
        77.75,
        50.76,
        78.76
      ],
      [
        "Reasoning",
        14.71,
        77.62,
        14.53,
        75.76
      ],
      [
        "Overall",
        22.18,
        76.07,
        37.03,
        76.01
      ]
    ],
    "c10": [
      [
        "Coding",
        18.76,
        70.88,
        34.61,
        75.14
      ],
      [
        "Co-work",
        31.65,
        82.13,
        51.3,
        82.14
      ],
      [
        "Reasoning",
        11.07,
        83.36,
        12.67,
        83.81
      ],
      [
        "Overall",
        21.71,
        77.57,
        35.87,
        79.5
      ]
    ]
  },
  "astra_truncation": [
    [
      "Coding",
      -2.69,
      57.14,
      10.34,
      75.28
    ],
    [
      "Co-work",
      40.05,
      77.09,
      21.36,
      75.21
    ],
    [
      "Reasoning",
      38.25,
      79.63,
      29.36,
      76.7
    ],
    [
      "Overall",
      21.35,
      69.08,
      18.35,
      75.56
    ]
  ],
  "components": [
    [
      "Astra",
      "scientific_code",
      "coding",
      12,
      -3.04,
      -3.54,
      3.34,
      -0.06,
      -1.59,
      -6.37,
      3.84,
      -1.6
    ],
    [
      "Astra",
      "software_contracts",
      "coding",
      12,
      1.45,
      -3.2,
      3.18,
      -0.08,
      -4.84,
      -6.37,
      3.61,
      -1.65
    ],
    [
      "Astra",
      "multilingual",
      "coding",
      8,
      -0.78,
      -2.4,
      2.85,
      -0.03,
      -5.71,
      -5.78,
      2.48,
      -1.22
    ],
    [
      "Astra",
      "codebase_tracing",
      "coding",
      8,
      1.88,
      -2.48,
      2.7,
      -0.06,
      0.0,
      -5.78,
      1.94,
      -1.32
    ],
    [
      "Astra",
      "gdp_data",
      "co-work",
      8,
      31.94,
      -10.92,
      16.48,
      1.47,
      12.57,
      -1.84,
      10.61,
      3.03
    ],
    [
      "Astra",
      "workflow_rules",
      "co-work",
      16,
      2.93,
      -14.25,
      25.9,
      3.19,
      3.47,
      -2.51,
      13.82,
      4.77
    ],
    [
      "Astra",
      "professional_memos",
      "co-work",
      4,
      -13.47,
      -8.13,
      9.71,
      0.64,
      -2.81,
      -0.99,
      7.1,
      1.8
    ],
    [
      "Astra",
      "carried_context",
      "co-work",
      4,
      -1.68,
      -8.21,
      8.83,
      0.68,
      -0.27,
      -0.99,
      7.11,
      1.79
    ],
    [
      "Astra",
      "arxiv_math",
      "reasoning",
      16,
      -3.04,
      -2.75,
      4.86,
      0.76,
      -4.14,
      -4.91,
      4.77,
      0.06
    ],
    [
      "Astra",
      "hle_choice",
      "reasoning",
      16,
      2.12,
      -2.72,
      5.55,
      0.79,
      -0.15,
      -5.36,
      5.23,
      0.01
    ],
    [
      "Astra",
      "constructed_math",
      "reasoning",
      8,
      6.75,
      -2.15,
      2.7,
      0.24,
      9.37,
      -3.53,
      2.3,
      -0.19
    ],
    [
      "GLM5.3",
      "implementation",
      "coding",
      33,
      -5.12,
      -14.18,
      22.61,
      0.95,
      2.15,
      -8.13,
      16.0,
      0.83
    ],
    [
      "GLM5.3",
      "debugging",
      "coding",
      5,
      1.46,
      -3.46,
      3.3,
      -0.05,
      0.73,
      -7.49,
      3.92,
      -0.38
    ],
    [
      "GLM5.3",
      "instruction_constraints",
      "co-work",
      20,
      11.35,
      -12.59,
      11.72,
      0.61,
      10.34,
      -3.19,
      16.45,
      6.17
    ],
    [
      "GLM5.3",
      "numeric_deliverables",
      "co-work",
      10,
      -5.76,
      -5.87,
      6.79,
      0.22,
      -2.11,
      -4.8,
      7.61,
      0.66
    ],
    [
      "GLM5.3",
      "arxiv_math",
      "reasoning",
      12,
      -1.92,
      -5.15,
      11.37,
      1.47,
      -0.76,
      -3.66,
      8.9,
      1.56
    ],
    [
      "GLM5.3",
      "hle_choice",
      "reasoning",
      20,
      27.58,
      -6.36,
      22.73,
      4.48,
      18.09,
      -5.2,
      16.55,
      3.47
    ],
    [
      "K3",
      "hle_choice",
      "reasoning",
      16,
      8.01,
      -4.01,
      11.63,
      2.26,
      9.2,
      -5.06,
      10.12,
      1.83
    ],
    [
      "K3",
      "hle_exact",
      "reasoning",
      8,
      -0.78,
      -3.18,
      5.37,
      0.7,
      0.17,
      -4.0,
      5.24,
      0.4
    ],
    [
      "K3",
      "arxiv_math",
      "reasoning",
      8,
      -2.45,
      -3.34,
      5.25,
      0.64,
      -4.14,
      -4.14,
      5.53,
      0.39
    ],
    [
      "K3",
      "base_implementation",
      "coding",
      15,
      2.93,
      -4.26,
      4.86,
      0.01,
      4.9,
      -4.61,
      8.34,
      1.09
    ],
    [
      "K3",
      "hard_implementation",
      "coding",
      5,
      1.78,
      -2.05,
      2.95,
      0.02,
      -0.82,
      -3.41,
      5.18,
      0.38
    ],
    [
      "K3",
      "debugging",
      "coding",
      15,
      -2.29,
      -4.62,
      4.66,
      -0.07,
      0.0,
      -4.45,
      8.34,
      1.07
    ],
    [
      "K3",
      "output_prediction",
      "coding",
      10,
      -1.63,
      -3.4,
      4.15,
      0.02,
      0.0,
      -4.38,
      7.44,
      0.76
    ],
    [
      "K3",
      "instruction_constraints",
      "co-work",
      18,
      13.03,
      -11.98,
      12.39,
      0.44,
      14.05,
      3.37,
      18.59,
      9.84
    ],
    [
      "K3",
      "professional_rubrics",
      "co-work",
      12,
      -3.58,
      -7.87,
      9.06,
      0.21,
      5.37,
      0.5,
      13.6,
      6.61
    ]
  ],
  "astra_outcomes": {
    "Kimi K3": [
      38,
      2,
      0
    ],
    "Gemma 4 31B": [
      34,
      6,
      0
    ],
    "GPT-5.5": [
      35,
      1,
      4
    ],
    "GLM-5.2": [
      22,
      1,
      17
    ],
    "DeepSeek V4 Flash": [
      6,
      2,
      32
    ]
  },
  "termination": {
    "coding": 88,
    "co-work": 68,
    "reasoning": 200
  },
  "remaining_weight": {
    "coding": {
      "minimum": 0.12500000000000003,
      "mean": 0.8194444444444445,
      "maximum": 1.0
    },
    "co-work": {
      "minimum": 0.46875,
      "mean": 0.8125,
      "maximum": 1.0
    },
    "reasoning": {
      "minimum": 0.2,
      "mean": 0.5833333333333334,
      "maximum": 1.0
    }
  },
  "component_panel": [
    "development-01",
    "development-06",
    "development-08",
    "holdout-01",
    "holdout-02",
    "holdout-07",
    "holdout-11",
    "sealed-new-01",
    "sealed-new-03",
    "sealed-new-04",
    "sealed-new-05",
    "sealed-new-06"
  ],
  "component_columns": [
    "researcher",
    "component",
    "domain",
    "questions",
    "score_effect",
    "score_random_low",
    "score_random_high",
    "score_random_mean",
    "pa_effect_pp",
    "pa_random_low_pp",
    "pa_random_high_pp",
    "pa_random_mean_pp"
  ]
}