Preprint · 2026

Lost in Benchmarks? Rethinking Large Language Model Benchmarking with Item Response Theory

Hongli Zhou, Hui Huang, Ziqing Zhao, Lvyuan Han, Huicheng Wang, Kehai Chen, Muyun Yang, Wei Bao, Jian Dong, Bing Xu, Conghui Zhu, Hailong Cao, Tiejun Zhao

BibTeX Copy

@misc{zhou2025lostbenchmarks,
	title = {Lost in {Benchmarks}? {Rethinking} {Large} {Language} {Model} {Benchmarking} with {Item} {Response} {Theory}},
	url = {https://arxiv.org/abs/2505.15055},
	publisher = {arXiv},
	author = {Zhou, Hongli and Huang, Hui and Zhao, Ziqing and Han, Lvyuan and Wang, Huicheng and Chen, Kehai and Yang, Muyun and Bao, Wei and Dong, Jian and Xu, Bing and Zhu, Conghui and Cao, Hailong and Zhao, Tiejun},
	year = {2026},
	note = {Note: Accepted to AAAI 2026},
	keywords = {replichai},
}