@inproceedings{kim2025uncertainty,title={On the Effect of Uncertainty on Layer-wise Inference Dynamics},author={Kim, Sunwoo and Yoo, Haneul and Oh, Alice},booktitle={ICML 2025 Workshop on Actionable Interpretability},year={2025},month=jul,}
Evaluating the performance and biases of large language models (LLMs) through role-playing scenarios is becoming increasingly common, as LLMs often exhibit biased behaviors in these contexts. Building on this line of research, we introduce PapersPlease, a benchmark consisting of 3,700 moral dilemmas designed to investigate LLMs’ decision-making in prioritizing various levels of human needs. In our setup, LLMs act as immigration inspectors deciding whether to approve or deny entry based on the short narratives of people. These narratives are constructed using the Existence, Relatedness, and Growth (ERG) theory, which categorizes human needs into three hierarchical levels. Our analysis of six LLMs reveals statistically significant patterns in decision-making, suggesting that LLMs encode implicit preferences. Additionally, our evaluation of the impact of incorporating social identities into the narratives shows varying responsiveness based on both motivational needs and identity cues, with some models exhibiting higher denial rates for marginalized identities. All data is publicly available at https://github.com/yeonsuuuu28/papers-please.
@inproceedings{myung-etal-2025-papersplease,title={{P}apers{P}lease: A Benchmark for Evaluating Motivational Values of Large Language Models Based on {ERG} Theory},author={Myung, Junho and Park, Yeon Su and Kim, Sunwoo and Yoo, Shin and Oh, Alice},editor={Arviv, Ofir and Clinciu, Miruna and Dhole, Kaustubh and Dror, Rotem and Gehrmann, Sebastian and Habba, Eliya and Itzhak, Itay and Mille, Simon and Perlitz, Yotam and Santus, Enrico and Sedoc, Jo{\~a}o and Shmueli Scheuer, Michal and Stanovsky, Gabriel and Tafjord, Oyvind},booktitle={Proceedings of the Fourth Workshop on Generation, Evaluation and Metrics (GEM<sup>2</sup>)},month=jul,year={2025},location={Vienna, Austria},publisher={Association for Computational Linguistics},url={https://aclanthology.org/2025.gem-1.47/},pages={522--531},isbn={979-8-89176-261-9},}