Hobert, Sebastian
How Are You, Chatbot? Evaluating Chatbots in Educational Settings – Results of a Literature Review Proceedings Article
In: Pinkwart, Niels; Konert, Johannes (Hrsg.): DELFI 2019, S. 259–270, Gesellschaft für Informatik e.V., Bonn, 2019, ISBN: 978-3-88579-691-6.
Abstract | Links | BibTeX | Schlagwörter: A, chatbots, evaluation, pedagogical conversational agents, technology-enhanced learning
@inproceedings{Hobert2019,
title = {How Are You, Chatbot? Evaluating Chatbots in Educational Settings – Results of a Literature Review},
author = {Sebastian Hobert},
editor = {Niels Pinkwart and Johannes Konert},
url = {https://dx.doi.org/10.18420/delfi2019_289},
doi = {10.18420/delfi2019_289},
isbn = {978-3-88579-691-6},
year = {2019},
date = {2019-09-16},
booktitle = {DELFI 2019},
pages = {259–270},
publisher = {Gesellschaft für Informatik e.V.},
address = {Bonn},
abstract = {Evaluation studies are essential for determining the utilization of technology-enhanced learning systems. Prior research often focuses on evaluating specific factors like the technology adoption or usability aspects. However, it needs to be questioned if evaluating only specific factors is appropriate in each case. The aim of this research paper is to outline which methods are suited for evaluating technology-enhanced learning systems in interdisciplinary research domains. Specifically, we focus our analysis on pedagogical conversational agents – i.e. learning systems that interact with learners using natural language. For instance, in addition to technology acceptance, further factors like learning success are more important in this case. Based on this assumption, we analyze the current state-of-the-art literature of pedagogical conversational agents to identify evaluation objectives, procedures and measuring instruments. Afterward, we use the results to propose a guideline for evaluations of pedagogical conversational agents.},
keywords = {A, chatbots, evaluation, pedagogical conversational agents, technology-enhanced learning},
pubstate = {published},
tppubtype = {inproceedings}
}
Bruechner, Dominik; Renz, Jan; Klingbeil, Mandy
Creating a Framework for User-Centered Development and Improvement of Digital Education Proceedings Article
In: Proceedings of the Sixth (2019) ACM Conference on Learning @ Scale, Association for Computing Machinery, New York, NY, USA, 2019, ISBN: 9781450368049.
Abstract | Links | BibTeX | Schlagwörter: evaluation, HPI Schul-Cloud, learning platform, O, user experience, user research framework, user-centered design
@inproceedings{Bruechner2019,
title = {Creating a Framework for User-Centered Development and Improvement of Digital Education},
author = {Dominik Bruechner and Jan Renz and Mandy Klingbeil},
url = {https://doi.org/10.1145/3330430.3333644},
doi = {10.1145/3330430.3333644},
isbn = {9781450368049},
year = {2019},
date = {2019-06-01},
booktitle = {Proceedings of the Sixth (2019) ACM Conference on Learning @ Scale},
number = {31},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {We investigate how the technology acceptance and learning experience of the digital education platform HPI Schul-Cloud (HPI School Cloud) for German secondary school teachers can be improved by proposing a user-centered research and development framework. We highlight the importance of developing digital learning technologies in a user-centered way to take differences in the requirements of educators and students into account. We suggest applying qualitative and quantitative methods to build a solid understanding of a learning platform's users, their needs, requirements, and their context of use. After concept development and idea generation of features and areas of opportunity based on the user research, we emphasize on the application of a multi-attribute utility analysis decision-making framework to prioritize ideas rationally, taking results of user research into account. Afterward, we recommend applying the principle build-learn-iterate to build prototypes in different resolutions while learning from user tests and improving the selected opportunities. Last but not least, we propose an approach for continuous short- and long-term user experience controlling and monitoring, extending existing web- and learning analytics metrics.},
keywords = {evaluation, HPI Schul-Cloud, learning platform, O, user experience, user research framework, user-centered design},
pubstate = {published},
tppubtype = {inproceedings}
}
Hessler, Michael; Pöpping, Daniel M; Hollstein, Hanna; Ohlenburg, Hendrik; Arnemann, Philip H; Massoth, Christina; Seidel, Laura M; Zarbock, Alexander; Wenk, Manuel
Availability of cookies during an academic course session affects evaluation of teaching Artikel
In: Medical Education, Bd. 52, Nr. 10, S. 1064–1072, 2018, ISSN: 1365-2923.
Abstract | Links | BibTeX | Schlagwörter: A, cookies, evaluation, influence, teaching
@article{Hessler2018,
title = {Availability of cookies during an academic course session affects evaluation of teaching},
author = {Michael Hessler and Daniel M Pöpping and Hanna Hollstein and Hendrik Ohlenburg and Philip H Arnemann and Christina Massoth and Laura M Seidel and Alexander Zarbock and Manuel Wenk},
url = {https://doi.org/10.1111/medu.13627},
doi = {10.1111/medu.13627},
issn = {1365-2923},
year = {2018},
date = {2018-06-29},
urldate = {2018-11-19},
journal = {Medical Education},
volume = {52},
number = {10},
pages = {1064–1072},
abstract = {Objectives Results from end-of-course student evaluations of teaching (SETs) are taken seriously by faculties and form part of a decision base for the recruitment of academic staff, the distribution of funds and changes to curricula. However, there is some doubt as to whether these evaluation instruments accurately measure the quality of course content, teaching and knowledge transfer. We investigated whether the provision of chocolate cookies as a content-unrelated intervention influences SET results. Methods We performed a randomised controlled trial in the setting of a curricular emergency medicine course. Participants were 118 third-year medical students. Participants were randomly allocated into 20 groups, 10 of which had free access to 500 g of chocolate cookies during an emergency medicine course session (cookie group) and 10 of which did not (control group). All groups were taught by the same teachers. Educational content and course material were the same for both groups. After the course, all students were asked to complete a 38-question evaluation form. Results A total of 112 students completed the evaluation form. The cookie group evaluated teachers significantly better than the control group (113.4 ± 4.9 versus 109.2 ± 7.3; p = 0.001, effect size 0.68). Course material was considered better (10.1 ± 2.3 versus 8.4 ± 2.8; p = 0.001, effect size 0.66) and summation scores evaluating the course overall were significantly higher (224.5 ± 12.5 versus 217.2 ± 16.1; p = 0.008, effect size 0.51) in the cookie group. Conclusions The provision of chocolate cookies had a significant effect on course evaluation. These findings question the validity of SETs and their use in making widespread decisions within a faculty.},
keywords = {A, cookies, evaluation, influence, teaching},
pubstate = {published},
tppubtype = {article}
}
Wiemeyer, Josef
Die eigene Lehre erforschen – Lohnt sich der Aufwand? Artikel
In: eleed, Bd. 12, Nr. 2, 2018, ISSN: 1860-7470.
Abstract | Links | BibTeX | Schlagwörter: A, Audience-Response-Systeme, e-learning, evaluation, Forschendes Lehren, Interaktivität, Pädagogische Agenten, Selbstgesteuertes Lernen, Sportwissenschaften, Studium
@article{Wiemeyer2018jfhm,
title = {Die eigene Lehre erforschen – Lohnt sich der Aufwand?},
author = {Josef Wiemeyer},
url = {http://nbn-resolving.de/urn:nbn:de:0009-5-46586},
issn = {1860-7470},
year = {2018},
date = {2018-03-01},
journal = {eleed},
volume = {12},
number = {2},
institution = {Fernuniversität in Hagen},
abstract = {Digitale Medien bieten nicht nur neue Optionen für Lehr-Lernprozesse. Vielmehr werden durch digitale Medien häufig auch Daten generiert, die wertvolle Informationen über das Lernverhalten liefern können, welche ohne Einsatz dieser Medien nicht verfügbar wären. Allerdings sind diese „Oberflächendaten“ schwer zu interpretieren. An vier Beispielen (Interaktivität, Selbstreguliertes Lernen, Audience-Response-System mit Lernspielen und Pädagogische Agenten) soll in diesem Beitrag die Ambivalenz mediendidaktischer Feldforschung aufgezeigt werden: Es zeigten sich einerseits die üblichen Beschränkungen dieser Art von Forschung (z. B. Bedingungskontrolle, fehlende Randomisierung, Selbstselektion und Konfundierungen), andererseits konnte die differenzielle Wirkung der oben erwähnten Medien offengelegt werden. Dem erhöhten Aufwand an Personal, Material und Zeit steht ein Erkenntnisgewinn bzgl. der Effekte des eigenen Lehrens gegenüber, der zur weiteren Verbesserung genutzt werden kann.},
howpublished = {Tagungsband des Jungen Forums für Medien und Hochschulentwicklung 2016},
type = {Keynote},
keywords = {A, Audience-Response-Systeme, e-learning, evaluation, Forschendes Lehren, Interaktivität, Pädagogische Agenten, Selbstgesteuertes Lernen, Sportwissenschaften, Studium},
pubstate = {published},
tppubtype = {article}
}
Finardi, Kyria Rebeca; Leao, Roberta Gomes; Amorim, Gabriel Brito
Mobile Assisted Language Learning: Affordances and Limitations of Duolingo Artikel
In: Education and Linguistics Research, Bd. 2, Nr. 2, 2016, ISSN: 2377-1356.
Abstract | Links | BibTeX | Schlagwörter: A, Duolingo, evaluation, language learning, mobile learning
@article{Finardi2016,
title = {Mobile Assisted Language Learning: Affordances and Limitations of Duolingo},
author = {Kyria Rebeca Finardi and Roberta Gomes Leao and Gabriel Brito Amorim},
url = {https://doi.org/10.5296/elr.v2i2.9842},
doi = {10.5296/elr.v2i2.9842},
issn = {2377-1356},
year = {2016},
date = {2016-09-11},
urldate = {2018-04-22},
journal = {Education and Linguistics Research},
volume = {2},
number = {2},
abstract = {This study aimed at analyzing how mobile assisted language learning (MALL) may be used to aid second/foreign language (L2) learning. So as to reach this goal, the Duolingo platform, one of the most popular apps among L2 learners, was analyzed in terms of its possibilities and limitations for L2 learning as perceived by its users. With that aim, a study was carried out with 80 participants, 45 of whom were L2 learners who answered an online survey about their preferences for L2 learning, 25 MALL users and 10 L2 teachers who were interviewed. Data analysis used a mixed methods design with predominantly qualitative data. Results of the study showed that participants recognize the relevance of MALL for L2 learning and think that the Duolingo may aid L2 learning to a certain extent, but agree that a teacher is necessary to foster interaction and the development of production skills in L2. Based on these results the study concluded that L2 teachers should consider using blended approaches combining MALL in traditional classes.},
keywords = {A, Duolingo, evaluation, language learning, mobile learning},
pubstate = {published},
tppubtype = {article}
}
Blackler, Alethea L.; Gomez, Rafael; Popovic, Vesna; Thompson, Helen M.
In: Interacting with Computers, Bd. 28, Nr. 1, S. 27–46, 2016, ISSN: 0953-5438.
Abstract | Links | BibTeX | Schlagwörter: A, evaluation, features, Ig-Nobel-Preis, manuals, RTFM, Usability, user guides
@article{Blackler2016,
title = {Life Is Too Short to RTFM: How Users Relate to Documentation and Excess Features in Consumer Products},
author = {Alethea L. Blackler and Rafael Gomez and Vesna Popovic and Helen M. Thompson},
url = {http://dx.doi.org/10.1093/iwc/iwu023},
doi = {10.1093/iwc/iwu023},
issn = {0953-5438},
year = {2016},
date = {2016-01-01},
urldate = {2018-10-20},
journal = {Interacting with Computers},
volume = {28},
number = {1},
pages = {27–46},
abstract = {This paper addresses two common problems that users of various products and interfaces encounter—over-featured interfaces and product documentation. Over-featured interfaces are seen as a problem as they can confuse and over-complicate everyday interactions. Researchers also often claim that users do not read product documentation, although they are often exhorted to ‘RTFM’ (read the field manual). We conducted two sets of studies with users which looked at the issues of both manuals and excess features with common domestic and personal products. The quantitative set was a series of questionnaires administered to 170 people over 7 years. The qualitative set consisted of two 6-month longitudinal studies based on diaries and interviews with a total of 15 participants. We found that manuals are not read by the majority of people, and most do not use all the features of the products that they own and use regularly. Men are more likely to do both than women, and younger people are less likely to use manuals than middle-aged and older ones. More educated people are also less likely to read manuals. Over-featuring and being forced to consult manuals also appears to cause negative emotional experiences. Implications of these findings are discussed.},
keywords = {A, evaluation, features, Ig-Nobel-Preis, manuals, RTFM, Usability, user guides},
pubstate = {published},
tppubtype = {article}
}