{
	"sources": {
		"aaru": {
			"questions": 2993,
			"protocol": "Full retrospective evaluation; self-assessed TVD before reference outcomes, without confidence-model fitting on this benchmark."
		},
		"simile": {
			"title": "Building confidence in Simile",
			"authors": [
				"Andrew Wesel",
				"Sarah Chen",
				"Percy Liang"
			],
			"url": "https://www.simile.com/blog/confidence",
			"accessed": "2026-09-15",
			"questionsApproximate": 8600,
			"protocol": "Pooled five-fold held-out predictions. Question text and entire experimental samples remain in the same fold. Model selection uses training/validation splits within each fold.",
			"reported": [
				{
					"id": "simile-cft",
					"method": "Simulation model fine-tune",
					"auc": 0.736,
					"pearson": 0.565,
					"rmse": 0.0776
				},
				{
					"id": "simile-probe",
					"method": "Hidden-state probe",
					"auc": 0.73,
					"pearson": 0.538,
					"rmse": 0.0792
				},
				{
					"id": "simile-stock",
					"method": "General-purpose model fine-tune",
					"auc": 0.72,
					"pearson": 0.538,
					"rmse": 0.079
				},
				{
					"id": "simile-embedding",
					"method": "Embedding regression",
					"auc": 0.686,
					"pearson": 0.446,
					"rmse": 0.0843
				},
				{
					"id": "simile-basic",
					"method": "Basic-feature regression",
					"auc": 0.566,
					"pearson": 0.122,
					"rmse": 0.0992
				}
			]
		}
	},
	"metric": {
		"auc": "Observed TVD below 16 points vs. at least 16, matching Wesel, Chen, and Liang. Lower predicted error identifies the below-threshold event. Average ranks give tied predictions half credit.",
		"pearson": "Correlation between predicted and observed per-question TVD.",
		"rmse": "Root mean squared difference between predicted and observed TVD, in points on a 0–100 scale. Values reported by Wesel, Chen, and Liang are multiplied by 100.",
		"weighting": "Equal question weight for Aaru; Wesel, Chen, and Liang report pooled held-out metrics.",
		"threshold": 16,
		"tolerance": 1e-9
	},
	"sensitivity": {
		"exactThresholdQuestions": 19,
		"strictlyAbove16Auc": 0.8481719322493578,
		"strictlyAbove10Auc": 0.8251814882032668
	},
	"rows": [
		{
			"id": "aaru",
			"name": "Aaru",
			"method": "Confidence predictor",
			"auc": 0.844920891566443,
			"pearson": 0.5947044581731762,
			"rmse": 5.322522299741567
		},
		{
			"id": "simile-cft",
			"method": "Simulation model fine-tune",
			"auc": 0.736,
			"pearson": 0.565,
			"rmse": 7.76,
			"name": "Wesel, Chen, and Liang"
		},
		{
			"id": "simile-probe",
			"method": "Hidden-state probe",
			"auc": 0.73,
			"pearson": 0.538,
			"rmse": 7.920000000000001,
			"name": "Wesel, Chen, and Liang"
		},
		{
			"id": "simile-stock",
			"method": "General-purpose model fine-tune",
			"auc": 0.72,
			"pearson": 0.538,
			"rmse": 7.9,
			"name": "Wesel, Chen, and Liang"
		},
		{
			"id": "simile-embedding",
			"method": "Embedding regression",
			"auc": 0.686,
			"pearson": 0.446,
			"rmse": 8.43,
			"name": "Wesel, Chen, and Liang"
		},
		{
			"id": "simile-basic",
			"method": "Basic-feature regression",
			"auc": 0.566,
			"pearson": 0.122,
			"rmse": 9.92,
			"name": "Wesel, Chen, and Liang"
		}
	]
}
