{
  "paperTitle": "EvalResearchBench: Can AI Agents Design Their Own Evaluations?",
  "paperUrl": "https://arxiv.org/pdf/2610.04184",
  "sourceRevision": "d0507960aadae9dfaee115fa2f9bc99c9327afa2",
  "domains": [
    "coding",
    "cowork",
    "reasoning",
    "overall"
  ],
  "groups": {
    "development": 6,
    "sealed": 8,
    "overall": 14
  },
  "researchers": 9,
  "candidates": 13,
  "rows": [
    {
      "id": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "kind": "agent",
      "score": {
        "development": [
          -5.19,
          40.41,
          60.64,
          20.98
        ],
        "sealed": [
          4.79,
          43.16,
          24.69,
          24.15
        ],
        "overall": [
          -0.2,
          42.06,
          36.67,
          22.79
        ]
      },
      "pa": {
        "development": [
          55.71,
          75.76,
          83.33,
          66.99
        ],
        "sealed": [
          60.32,
          79.47,
          76.09,
          71.44
        ],
        "overall": [
          58.01,
          77.98,
          78.51,
          69.54
        ]
      }
    },
    {
      "id": "grok-4-7",
      "name": "Grok 4.7",
      "kind": "agent",
      "score": {
        "development": [
          24.35,
          10.34,
          13.33,
          17.84
        ],
        "sealed": [
          22.22,
          37.84,
          14.48,
          26.14
        ],
        "overall": [
          23.28,
          26.84,
          14.1,
          22.59
        ]
      },
      "pa": {
        "development": [
          77.49,
          72.1,
          82.22,
          76.48
        ],
        "sealed": [
          66.85,
          81.63,
          71.93,
          73.66
        ],
        "overall": [
          72.17,
          77.81,
          75.36,
          74.87
        ]
      }
    },
    {
      "id": "fable-5-1",
      "name": "Fable 5.1",
      "kind": "agent",
      "score": {
        "development": [
          9.81,
          23.52,
          63.94,
          23.4
        ],
        "sealed": [
          20.92,
          19.68,
          22.6,
          20.87
        ],
        "overall": [
          15.36,
          21.22,
          36.38,
          21.96
        ]
      },
      "pa": {
        "development": [
          61.03,
          69.87,
          81.11,
          67.32
        ],
        "sealed": [
          63.94,
          73.13,
          67.05,
          68.16
        ],
        "overall": [
          62.48,
          71.82,
          71.73,
          67.8
        ]
      }
    },
    {
      "id": "opus-5",
      "name": "Opus 5",
      "kind": "agent",
      "score": {
        "development": [
          -6.36,
          24.34,
          77.99,
          17.93
        ],
        "sealed": [
          5.75,
          47.6,
          18.7,
          24.68
        ],
        "overall": [
          -0.31,
          38.3,
          38.46,
          21.79
        ]
      },
      "pa": {
        "development": [
          51.26,
          69.02,
          80.0,
          61.97
        ],
        "sealed": [
          56.3,
          74.61,
          71.89,
          67.06
        ],
        "overall": [
          53.78,
          72.37,
          74.59,
          64.88
        ]
      }
    },
    {
      "id": "kimi-k3",
      "name": "Kimi K3",
      "kind": "agent",
      "score": {
        "development": [
          6.49,
          12.34,
          55.33,
          16.58
        ],
        "sealed": [
          7.93,
          24.16,
          28.35,
          19.12
        ],
        "overall": [
          7.21,
          19.43,
          37.35,
          18.03
        ]
      },
      "pa": {
        "development": [
          64.33,
          74.07,
          81.11,
          70.38
        ],
        "sealed": [
          70.88,
          87.07,
          73.99,
          77.73
        ],
        "overall": [
          67.61,
          81.87,
          76.36,
          74.58
        ]
      }
    },
    {
      "id": "gpt-5-6-sol",
      "name": "GPT-5.6 Sol",
      "kind": "agent",
      "score": {
        "development": [
          21.9,
          -6.66,
          50.45,
          17.14
        ],
        "sealed": [
          17.71,
          10.24,
          25.58,
          16.88
        ],
        "overall": [
          19.81,
          3.48,
          33.87,
          16.99
        ]
      },
      "pa": {
        "development": [
          66.94,
          46.38,
          81.11,
          62.45
        ],
        "sealed": [
          60.6,
          60.31,
          74.54,
          63.98
        ],
        "overall": [
          63.77,
          54.74,
          76.73,
          63.32
        ]
      }
    },
    {
      "id": "glm-5-3",
      "name": "GLM-5.3",
      "kind": "agent",
      "score": {
        "development": [
          1.86,
          11.68,
          50.0,
          13.15
        ],
        "sealed": [
          8.73,
          18.3,
          19.91,
          15.12
        ],
        "overall": [
          5.29,
          15.65,
          29.94,
          14.27
        ]
      },
      "pa": {
        "development": [
          56.67,
          67.51,
          72.22,
          62.87
        ],
        "sealed": [
          53.24,
          77.57,
          68.18,
          66.1
        ],
        "overall": [
          54.95,
          73.55,
          69.53,
          64.72
        ]
      }
    },
    {
      "id": "deepseek-v4-pro",
      "name": "DeepSeek V4 Pro",
      "kind": "agent",
      "score": {
        "development": [
          3.2,
          5.55,
          44.0,
          10.78
        ],
        "sealed": [
          -0.53,
          11.59,
          24.51,
          10.28
        ],
        "overall": [
          1.33,
          9.17,
          31.01,
          10.49
        ]
      },
      "pa": {
        "development": [
          56.78,
          64.27,
          74.44,
          62.22
        ],
        "sealed": [
          63.19,
          65.91,
          64.18,
          64.46
        ],
        "overall": [
          59.99,
          65.25,
          67.6,
          63.5
        ]
      }
    },
    {
      "id": "fable-5-imputed",
      "name": "Fable 5 (imputed)",
      "kind": "scenario",
      "score": {
        "development": [
          -7.77,
          -12.58,
          -18.63,
          -11.18
        ],
        "sealed": [
          -5.68,
          -21.07,
          -18.58,
          -14.68
        ],
        "overall": [
          -6.72,
          -17.67,
          -18.6,
          -13.18
        ]
      },
      "pa": {
        "development": [
          44.42,
          44.78,
          37.78,
          43.43
        ],
        "sealed": [
          53.47,
          38.13,
          26.81,
          41.06
        ],
        "overall": [
          48.95,
          40.79,
          30.47,
          42.07
        ]
      }
    },
    {
      "id": "random120",
      "name": "Random120",
      "kind": "baseline",
      "score": {
        "development": [
          21.44,
          -10.71,
          12.79,
          9.28
        ],
        "sealed": [
          24.39,
          13.39,
          7.15,
          15.96
        ],
        "overall": [
          22.92,
          3.75,
          9.03,
          13.1
        ]
      },
      "pa": {
        "development": [
          73.87,
          55.96,
          68.06,
          66.93
        ],
        "sealed": [
          76.63,
          62.06,
          69.2,
          69.31
        ],
        "overall": [
          75.25,
          59.62,
          68.82,
          68.29
        ]
      }
    }
  ],
  "matchedBaseline": {
    "candidates": 12,
    "scoreWins": 7,
    "paWins": 4,
    "completeSuites": 8,
    "rows": [
      {
        "name": "Astra",
        "score": 21.35,
        "pa": 69.08
      },
      {
        "name": "Grok 4.7",
        "score": 22.18,
        "pa": 76.07
      },
      {
        "name": "Fable 5.1",
        "score": 20.48,
        "pa": 68.46
      },
      {
        "name": "Opus 5",
        "score": 20.16,
        "pa": 64.81
      },
      {
        "name": "Kimi K3",
        "score": 17.72,
        "pa": 74.17
      },
      {
        "name": "Sol",
        "score": 17.13,
        "pa": 62.55
      },
      {
        "name": "GLM-5.3",
        "score": 14.15,
        "pa": 65.01
      },
      {
        "name": "DeepSeek V4 Pro",
        "score": 10.45,
        "pa": 63.51
      },
      {
        "name": "Random120",
        "score": 13.1,
        "pa": 68.29
      }
    ]
  },
  "weights": {
    "candidates": 12,
    "questions": 20,
    "dominantQuestions": 6,
    "originalMass": 93.87,
    "uniformMass": 30,
    "original": {
      "score": 22.18,
      "pa": 76.07
    },
    "uniform": {
      "score": 37.03,
      "pa": 76.01
    }
  }
}