Beatriz Barroso Gstrein, MDS

@inproceedings{10.1145/3774935.3806172,
author = {M\"{u}ller, Michael and Mohammadi, Amir Reza and Peintner, Andreas and Barroso Gstrein, Beatriz and Specht, G\"{u}nther and Zangerle, Eva},
title = {Can Third-Party Annotators Reliably Evaluate Conversational Recommender Systems?},
year = {2026},
isbn = {9798400723117},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3774935.3806172},
doi = {10.1145/3774935.3806172},
abstract = {User-centric evaluation has become a key paradigm for assessing Conversational Recommender Systems (CRS), aiming to capture subjective qualities such as satisfaction, trust, and rapport. To enable scalable evaluation, recent work increasingly relies on third-party annotations of static dialogue logs by crowd workers or large language models. However, the reliability of this practice remains largely unexamined. In this paper, we present a large-scale empirical study investigating the reliability and structure of user-centric CRS evaluation on static dialogue transcripts. We collected 1,053 annotations from 124 crowd workers on 200 ReDial dialogues using the 18-dimensional CRS-Que framework. Using random-effects reliability models and correlation analysis, we quantify the stability of individual dimensions and their interdependencies. Our results show that utilitarian and outcome-oriented dimensions such as accuracy, usefulness, and satisfaction achieve moderate reliability under aggregation, whereas socially grounded constructs such as humanness and rapport are substantially less reliable. Furthermore, many dimensions collapse into a single global quality signal, revealing a strong halo effect in third-party judgments. These findings challenge the validity of single-annotator and LLM-based evaluation protocols and motivate the need for multi-rater aggregation and dimension reduction in offline CRS evaluation.},
booktitle = {Proceedings of the 34th ACM Conference on User Modeling, Adaptation and Personalization},
pages = {351–355},
numpages = {5},
keywords = {Conversational Recommender Systems, User-Centric, Evaluation, User Interaction, Personalization},
location = {
},
series = {UMAP '26}
}

@article{mohammadi2025quantitative,
  title={Yearning for Better Metrics: Revisiting the ReDial Dataset for Evaluating Conversational Recommender Systems},
  author={M{\"u}ller, Michael M and Mohammadi, Amir Reza and Peintner, Andreas and Gstrein, Beatriz Barroso and Zangerle, Eva and Specht, G{\"u}nther},
  year={2025},
  booktitle={Proceedings of the 36th GI-Workshop Grundlagen von Datenbanken (GvDB'25), Regensburg, Germany},
}

@article{mohammadi2025quantitative,
  title={Quantitative Assessment of GNN Counterfactual Explanation Robustness and Reproducibility under Adversarial Influence},
  author={Mohammadi, Amir Reza and M{\"u}ller, Michael M and Peintner, Andreas and Gstrein, Beatriz Barroso and Zangerle, Eva and Specht, G{\"u}nther},
  year={2025},
  booktitle    = {Proceedings of the 36th GI-Workshop Grundlagen von Datenbanken (GvDB'25), Regensburg, Germany},
}