Journal of multidisciplinary evaluation最新文献

{"title":"Glocal Evaluation Competencies for Learning As We Go: Zooming in and zooming out to connect system-level solutions to local beneficiaries","authors":"Keiko Kuji-Shikatani, Charmagne Campbell-Patton, Wendy Rowe","doi":"10.56645/jmde.v19i46.881","DOIUrl":"https://doi.org/10.56645/jmde.v19i46.881","url":null,"abstract":"Identifying essential competencies for evaluators has received significant attention in recent years yet practical examples of how to apply competencies to real-time learning in complex environments are lacking. In particular, the experience of those at the local level - ultimate beneficiary individuals (UBIs) - can get lost when evaluations take a systems perspective. Experienced evaluators share how Learning as we go is used to describe utilization-focused developmental evaluation embedding evaluative thinking and building capacity in public sector programs, that support learning and adaptation to improve the lives of those most impacted by inequitable and unsustainable global systems.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"230 2","pages":""},"PeriodicalIF":0.0,"publicationDate":"2023-12-11","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"138981383","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"In Plain Sight or Just Plain Obscured?: A Review of Professional Evaluation Associations’ Frameworks for Evaluation Practice Supporting Equity, Diversity and Inclusion (EDI)","authors":"Jane Whynot","doi":"10.56645/jmde.v19i46.893","DOIUrl":"https://doi.org/10.56645/jmde.v19i46.893","url":null,"abstract":"With an increasing focus on integrating equity, diversity and inclusion (EDI) in evaluation practice and products, there is an accompanying need to examine what structural supports exist that are provided by professional evaluation associations. This contribution systematically examines the competencies of five professional evaluation associations from Africa, Australia, Europe and North America to identify how evaluators can align integrating EDI in thei evaluation practice to professional competency domains. Also offered is a summary of thoughts on training opportunities for integrating EDI given review findings. Professional evaluation association websites were reviewed during May through to July of 2022. Content was downloaded into an Excel spreadsheet, and organized for EDI review purposes by competency domains, subdomains which occurred during August and September of 2022. The presentation of EDI content in evaluator competencies was found to be highly varied; variations were found in tone/theme, principles, and domains and subdomains.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"4 1","pages":""},"PeriodicalIF":0.0,"publicationDate":"2023-12-11","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"138979459","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Evaluation Capture, Evaluator Resilience, and the Need for Competencies of Evaluators","authors":"Frans Leeuw, Lyn E. Pleger","doi":"10.56645/jmde.v19i46.863","DOIUrl":"https://doi.org/10.56645/jmde.v19i46.863","url":null,"abstract":"Evaluation capture is the situation in which evaluations and evaluators are surrounded by protocols, guidelines, standards, norms, criteria, templates, oversight and reviews from national and supranational organizations, governments, NGO’s and agencies that ask for and finance monitoring and evaluation activities; these protocols and guidelines are sometimes labelled or part of ‘evaluation policies.’ The paper describes several developments in evaluation that contribute to this capture, referred to as mechanisms. The paper also addresses the question how to tackle this issue. The focus is on the role of the evaluator’s resilience, while several competences of evaluators to realize that are also addressed.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"52 3","pages":""},"PeriodicalIF":0.0,"publicationDate":"2023-12-11","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"138982084","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Utilizing journey mapping to evaluate youth programs and social service systems: Case studies","authors":"Emma Connell, Amanda Petersen, Mongkol Teng","doi":"10.56645/jmde.v19i45.723","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.723","url":null,"abstract":"Background: Journey mapping is a relatively new, promising method for use in the evaluation sector. In this method, individuals who have gone through a program or system provide feedback on the process in a chronological way, highlighting successes and challenges they have encountered through the process. This article provides an overview of the method and three case studies describing how journey mapping has been used to evaluate youth programs and social service systems.
 Purpose: To aid evaluators in understanding journey mapping and the ways in which it can be applied to various evaluation projects with the intention to help them determine whether or not the method is appropriate in their practice.
 Setting: Journey mapping can be applied in evaluations examining how individuals and groups interact with programs, organizations, and systems.
 Data collection and analysis: The journey mapping method, including data collection through focus groups and interviews and thematic analysis of notes and transcripts.
 Findings: Journey mapping is a method that can illuminate successes and challenges individuals and families face when interacting with a program, organization, or system. Specific recommendations for study design, data collection, analysis, and reporting are offered for evaluators’ consideration.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"23 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437763","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"The Relationship Between Employee Motivation and Evaluation Capacity in a Community-Based Education Organization","authors":"Anuradha Sen, James C. Anderson, Thomas Archibald, Tiffany Drape","doi":"10.56645/jmde.v19i45.719","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.719","url":null,"abstract":"Background: Evaluation capacity building (ECB) has gained popularity among organizations due to the increased importance of accountability and organizational effectiveness. While the ECB literature has occasionally addressed the notion of motivation, it has usually been in terms of motivation to do or use evaluation (Clinton, 2014; Taylor-Ritzler et al., 2013); this study sought to ascertain whether general overall employee motivation in an organization is itself related to evaluation capacity. By better understanding this relationship, those who are involved in administering, implementing, evaluating, or researching ECB can be better equipped to understand one of the ‘mediating conditions’ or ‘antecedent conditions’ (Cousins et al., 2014) affecting an organization’s ability to do and use evaluation, and, in turn, can more efficiently and effectively craft their ECB work. Purpose: The purpose of this study was to explore the relationship between: (a) employee motivation and individual evaluation capacity; (b) employee motivation and evaluative thinking, and (c) evaluation capacity and evaluative thinking. Setting: The study focused on the Cooperative Extension System, a non-formal community-based education organization linked to public land-grant universities throughout the United States. Specifically, this study drew participants from two state Extension systems, Virginia and Maryland. Intervention: Not applicable. Research Design: This quantitative study used a descriptive correlational design (Creswell, 2003) to uncover the relationship between the variables: motivation and evaluation capacity, motivation and evaluative thinking, and evaluation capacity and evaluative thinking. Data Collection and Analysis: To investigate the relationship between the factors of interest (motivation, evaluation capacity, and evaluative thinking), three instruments were used: the Multidimensional Work Motivation Scale (MWMS), the Evaluation Capacity Assessment Instrument (ECAI), and the Evaluative Thinking Inventory (ETI). STATA MP 13.1 quantitative software was used to analyze the collected data. Findings: Employees with lower overall motivation in doing their work have lower evaluation capacity, and employees with higher motivation which is triggered by no external means but driven by internal factors have higher evaluation capacity.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"207 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437766","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Tracing the Transnational Influence of Jennifer Greene’s Ideas: Research on Evaluation Theory Across Countries","authors":"Melissa Goodnight, Cherie Avent, Ramya Kumaran","doi":"10.56645/jmde.v19i45.845","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.845","url":null,"abstract":"The study examines the transnational influence of American evaluation theorist Jennifer Greene’s ideas. Using mixed methods citation analysis, we investigate how Greene’s work in program evaluation theory on equity, democracy, and values engagement has impacted the field across countries. Rather than investigating theorist influence in a strictly quantitative way, we sought a contextualized, richer understanding of the contexts in which Greene’s ideas being used to observe the imprint of her work on evaluators theorizing and practicing outside of the United States. To achieve this depth of knowledge required both quantitative and qualitative strategies as well as attention to geographical and cultural locations and translations. This study details a distinct methodological strategy for conducting research on evaluation theory.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"42 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437759","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Excessive Evaluation Anxiety (XEA): The Last Two Decades","authors":"Nia Seunghye Kang, Katherine Moreau","doi":"10.56645/jmde.v19i45.701","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.701","url":null,"abstract":"Background: Excessive evaluation anxiety (XEA) refers to disproportionate or increased evaluation anxiety among those affected by evaluation (e.g., stakeholders) characterized by the sole presence of negative consequences. It can compromise evaluator-stakeholder relationships, presenting as a barrier for program evaluation. Moreover, XEA can both cause and be caused by resistance to evaluation, which is an interrelated topic that shares many common causes, characteristics, and mitigation strategies. The participatory and interactive nature of modern evaluation approaches can exacerbate the presence of XEA. However, researchers have not explored the current state of literature on XEA.
 Purpose: To explore the current state of the literature on XEA over the past 20 years.
 Setting: Not applicable.
 Intervention: Not applicable.
 Research Design: Literature review.
 Data Collection and Analysis: We conducted a literature search of Academic Search Complete, Web of Science, and Scopus. We complemented the database search by a journal search of the American Journal of Evaluation, Evaluation, and the Canadian Journal of Program Evaluation. We then conducted a thematic analysis of the articles that met the inclusion criteria.
 Findings: Upon review of the articles, we identified four main themes in the literature related to XEA. Specifically, XEA: leads to poor evaluator-stakeholder relationships; is influenced by cultural factors; can be mitigated through the development of interpersonal skills; and can be mitigated through a systematic and evidence-based approach to evaluation.
 Keywords: Excessive evaluation anxiety, resistance to evaluation, program evaluation, evaluator-stakeholder relationships","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"723 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437756","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"A Complexity-Based Plan for Evaluating Transformation","authors":"Jonny Morell","doi":"10.56645/jmde.v19i45.867","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.867","url":null,"abstract":"Abstract This article presents a case for more rigorous application of complexity science in our efforts to evaluate activity that seeks to bring about transformative change. It builds on the work that is already going on in the evaluation community. Three constructs from complexity science are employed – sensitive dependence, emergence, and social attractors. The paper argues that if–then logic is recommended for small-scale change within transformation efforts, but that to evaluate transformation writ large, data from if–then evaluation must be embedded in, and interpreted in terms of, complex behavior. Methodologies for evaluating within this framework are presented. The argument is linked to a definition of transformation that is multidimensional, non-linear, and measurable. The paper is built around a generic model of transformational change and shows how that model can be customized for specific transformation scenarios. It also shows how evaluation with respect to complexity can be accomplished with methodologies that are well known and well-practiced in the evaluation community.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"32 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437767","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"We Can’t Hear You – You’re on Mute: Findings From a Review of Evaluation Capacity Building (ECB) Practice Online","authors":"Ann Marie Castleman, Minji Cho, Isabelle Bourgeois, Leslie Fierro, Sebastian Lemire","doi":"10.56645/jmde.v19i45.739","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.739","url":null,"abstract":"Background: In her presidential address to the American Evaluation Association (AEA) in 2007, Hallie Preskill (2008) highlighted the potential role of technology to promote learning from evaluation, noting the increased use of computers, the internet, and social media as untapped ways to facilitate evaluation. More than ten years later in the context of the COVID-19 pandemic, evaluators and evaluation capacity building (ECB) practitioners found themselves needing to shift to online modalities to conduct evaluation and build capacity. The COVID-19 pandemic, technological advancements, and the rapid shift to remote work have changed our way of working (Gratton, 2021; Kane et al., 2021). Building evaluation capacity is no exception to this trend. Purpose: This study aimed to examine ways that practitioners have built evaluation capacity online or have used technology to do so, to capture lessons learned that can be applied in a COVID and post-normal context. Setting: Findings from this study can be applied in online contexts for developing evaluation capacity. Intervention: Not applicable. Research Design: The study design consisted of a rapid review of the ECB literature published from 2000 to 2019 in eight academic journals focused on evaluation research and practice. Data Collection and Analysis: Twenty-nine case applications of ECB practice that: 1) mentioned use of technology as a strategy for building evaluation capacity or 2) noted that at least one component of the ECB intervention was carried out online or virtually were reviewed for this study. Quantitative data were analyzed via descriptive statistics. Qualitative data were coded in MAXQDA using conventional content analysis (Hsieh & Shannon, 2005). Findings: More diverse online interventions have increased over time. Less than half (45%) of ECB interventions made use of both asynchronous and synchronous strategies for building capacity while more than one-third (38%) made use of asynchronous only strategies. Key barriers to implementing ECB strategies online included lack of social connections to other participants during the capacity building activity, technical malfunctions, lack of access to or familiarity with the technology in use, and limited resources for carrying out evaluation activities. Key facilitators for enhancing implementation included facilitating participant interaction and relationship-building both on and off-line, tailoring ECB activities to participant work contexts, and providing tutorials for accessing and using the technology in play.","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"418 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437765","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Editors' Note","authors":"Lyssa Becho, Michael A. Harnar","doi":"10.56645/jmde.v19i45.939","DOIUrl":"https://doi.org/10.56645/jmde.v19i45.939","url":null,"abstract":"","PeriodicalId":91909,"journal":{"name":"Journal of multidisciplinary evaluation","volume":"56 1","pages":"0"},"PeriodicalIF":0.0,"publicationDate":"2023-09-15","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135437757","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":0,"RegionCategory":"","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}