IEEE Transactions on Games最新文献

{"title":"Multigoal Reinforcement Learning via Exploring Entropy-Regularized Successor Matching","authors":"Xiaoyun Feng;Yun Zhou","doi":"10.1109/TG.2023.3304315","DOIUrl":"10.1109/TG.2023.3304315","url":null,"abstract":"Multigoal reinforcement learning (RL) algorithms tend to achieve and generalize over diverse goals. However, unlike single-goal agents, multigoal agents struggle to break through the exploration bottleneck with a fair share of interactions, owing to rarely reusable goal-oriented experiences with sparse goal-reaching rewards. Therefore, well-arranged behavior goals during training are essential for multigoal agents, especially in long-horizon tasks. To this end, we propose efficient multigoal exploration on the basis of maximizing the entropy of successor features and Exploring entropy-regularized successor matching, namely, E\u0000<inline-formula><tex-math>$^{2}$</tex-math></inline-formula>\u0000SM. E\u0000<inline-formula><tex-math>$^{2}$</tex-math></inline-formula>\u0000SM adopts the idea of a successor feature and extends it to entropy-regularized goal-reaching successor mapping that serves as a more stable state feature under sparse rewards. The key contribution of our work is to perform intrinsic goal setting with behavior goals that are more likely to be achieved in terms of future state occupancies as well as promising in expanding the exploration frontier. Experiments on challenging long-horizon manipulation tasks show that E\u0000<inline-formula><tex-math>$^{2}$</tex-math></inline-formula>\u0000SM deals well with sparse rewards and in pursuit of maximal state-covering, E\u0000<inline-formula><tex-math>$^{2}$</tex-math></inline-formula>\u0000SM efficiently identifies valuable behavior goals toward specific goal-reaching by matching the successor mapping.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"15 4","pages":"538-548"},"PeriodicalIF":2.3,"publicationDate":"2023-08-11","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"62570212","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Leveraging Joint-Action Embedding in Multiagent Reinforcement Learning for Cooperative Games","authors":"Xingzhou Lou;Junge Zhang;Yali Du;Chao Yu;Zhaofeng He;Kaiqi Huang","doi":"10.1109/TG.2023.3302694","DOIUrl":"10.1109/TG.2023.3302694","url":null,"abstract":"State-of-the-art multiagent policy gradient (MAPG) methods have demonstrated convincing capability in many cooperative games. However, the exponentially growing joint-action space severely challenges the critic's value evaluation and hinders performance of MAPG methods. To address this issue, we augment Central-Q policy gradient with a joint-action embedding function and propose mutual-information maximization MAPG (M3APG). The joint-action embedding function makes joint-actions contain information of state transitions, which will improve the critic's generalization over the joint-action space by allowing it to infer joint-actions' outcomes. We theoretically prove that with a fixed joint-action embedding function, the convergence of M3APG is guaranteed. Experiment results of the \u0000<italic>StarCraft</i>\u0000 multiagent challenge (SMAC) demonstrate that M3APG gives evaluation results with better accuracy and outperform other MAPG basic models across various maps of multiple difficulty levels. We empirically show that our joint-action embedding model can be extended to value-based multiagent reinforcement learning methods and state-of-the-art MAPG methods. Finally, we run an ablation study to show that the usage of mutual information in our method is necessary and effective.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 2","pages":"470-482"},"PeriodicalIF":2.3,"publicationDate":"2023-08-07","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"62570196","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Improved Exploration With Demonstrations in Procedurally-Generated Environments","authors":"Mao Xu;Shuzhi Sam Ge;Dongjie Zhao;Qian Zhao","doi":"10.1109/TG.2023.3299986","DOIUrl":"10.1109/TG.2023.3299986","url":null,"abstract":"Exploring sparse reward environments remains a major challenge in model-free deep reinforcement learning (RL). State-of-the-art exploration methods address this challenge by utilizing intrinsic rewards to guide exploration in uncertain environment dynamics or novel states. However, these methods fall short in procedurally-generated environments, where the agent is unlikely to visit a state more than once due to the different environments generated in each episode. Recently, imitation-learning-based exploration methods have been proposed to guide exploration in different kinds of procedurally-generated environments by imitating high-quality exploration episodes. However, these methods have weaker exploration capabilities and lower sample efficiency in complex procedurally-generated environments. Motivated by the fact that demonstrations can guide exploration in sparse reward environments, we propose improved exploration with demonstrations (IEWD), an improved imitation-learning-based exploration method in procedurally-generated environments, which utilizes demonstrations from these environments. IEWD assigns different episode-level exploration scores to each demonstration episode and generated episode. IEWD then ranks these episodes based on their scores and stores highly-scored episodes into a small ranking buffer. IEWD treats these highly-scored episodes as good exploration episodes and makes the deep RL agent imitate exploration behaviors from the ranking buffer to reproduce exploration behaviors from good exploration episodes. Additionally, IEWD adopts the experience replay buffer to store generated positive episodes and demonstrations and employs self-imitating learning to utilize experiences from the experience replay buffer to optimize the policy of the deep RL agent. We evaluate our method IEWD on several procedurally-generated MiniGrid environments and 3-D maze environments from MiniWorld. The results show that IEWD significantly outperforms existing learning from demonstration methods and exploration methods, including state-of-the-art imitation-learning-based exploration methods, in terms of sample efficiency and final performance in complex procedurally-generated environments.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 3","pages":"530-543"},"PeriodicalIF":1.7,"publicationDate":"2023-07-31","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"62570339","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Full DouZero+: Improving DouDizhu AI by Opponent Modeling, Coach-Guided Training and Bidding Learning","authors":"Youpeng Zhao;Jian Zhao;Xunhan Hu;Wengang Zhou;Houqiang Li","doi":"10.1109/TG.2023.3299612","DOIUrl":"10.1109/TG.2023.3299612","url":null,"abstract":"With the development of deep reinforcement learning, much progress in various perfect and imperfect information games has been achieved. Among these games, \u0000<italic>DouDizhu</i>\u0000, a popular card game in China, poses great challenges because of the imperfect information, large state and action space as well as the cooperation issue. In this article, we put forward an AI system for this game, which adopts opponent modeling and coach-guided training to help agents make better decisions when playing cards. Besides, we take the bidding phase of \u0000<italic>DouDizhu</i>\u0000 into consideration, which is usually ignored by existing works, and train a bidding network using Monte Carlo simulation. As a result, we achieve a full version of our AI system that is applicable to real-world competitions. We conduct extensive experiments to evaluate the effectiveness of the three techniques adopted in our method and demonstrate the superior performance of our AI over the state-of-the-art \u0000<italic>DouDizhu</i>\u0000 AI, i.e., DouZero. We upload our AI systems, one is bidding-free and the other is equipped with a bidding network, to Botzone platform and they both rank the first among over 400 and 250 AI programs on the two corresponding leaderboards, respectively.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 3","pages":"518-529"},"PeriodicalIF":1.7,"publicationDate":"2023-07-28","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"62570319","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Generating Interpretable Play-Style Descriptions Through Deep Unsupervised Clustering of Trajectories","authors":"Branden Ingram;Clint van Alten;Richard Klein;Benjamin Rosman","doi":"10.1109/TG.2023.3299074","DOIUrl":"10.1109/TG.2023.3299074","url":null,"abstract":"In any game, play style is a concept that describes the technique and strategy employed by a player to achieve a goal. Identifying a player's style is desirable as it can enlighten players on which approaches work better or worse in different scenarios and inform developers of the value of design decisions. In previous work, we demonstrated an unsupervised LSTM-autoencoder clustering approach for play-style identification capable of handling multidimensional variable length player trajectories. The efficacy of our model was demonstrated on both complete and partial trajectories in both a simulated and natural environment. Lastly, through state frequency analysis, the properties of each of the play styles were identified and compared. This work expands on this approach by demonstrating a process by which we utilize temporal information to identify the decision boundaries related to particular clusters. Additionally, we demonstrate further robustness by applying the same techniques to \u0000<italic>MiniDungeons</i>\u0000, another popular domain for player modeling research. Finally, we also propose approaches for determining mean play-style examples suitable for describing general play-style behaviors and for determining the correct number of represented play-styles.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"15 4","pages":"507-516"},"PeriodicalIF":2.3,"publicationDate":"2023-07-26","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"62570282","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Hierarchically Composing Level Generators for the Creation of Complex Structures","authors":"Michael Beukman;Manuel Fokam;Marcel Kruger;Guy Axelrod;Muhammad Nasir;Branden Ingram;Benjamin Rosman;Steven James","doi":"10.1109/TG.2023.3297619","DOIUrl":"https://doi.org/10.1109/TG.2023.3297619","url":null,"abstract":"Procedural content generation (PCG) is a growing field, with numerous applications in the video game industry and great potential to help create better games at a fraction of the cost of manual creation. However, much of the work in PCG is focused on generating relatively straightforward levels in simple games, as it is challenging to design an optimizable objective function for complex settings. This limits the applicability of PCG to more complex and modern titles, hindering its adoption in the industry. Our work aims to address this limitation by introducing a compositional level generation method that recursively composes simple low-level generators to construct large and complex creations. This approach allows for easily-optimizable objectives and the ability to design a complex structure in an interpretable way by referencing lower-level components. We empirically demonstrate that our method outperforms a noncompositional baseline by more accurately satisfying a designer's functional requirements in several tasks. Finally, we provide a qualitative showcase (in \u0000<italic>Minecraft</i>\u0000) illustrating the large and complex, but still coherent, structures that were generated using simple base generators.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 2","pages":"459-469"},"PeriodicalIF":2.3,"publicationDate":"2023-07-21","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"141333979","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Mouse Sensitivity in First-Person Targeting Tasks","authors":"Ben Boudaoud;Josef Spjut;Joohwan Kim","doi":"10.1109/TG.2023.3293692","DOIUrl":"https://doi.org/10.1109/TG.2023.3293692","url":null,"abstract":"Mouse sensitivity in first-person targeting tasks is a highly debated issue. Recommendations within a single game can vary by a factor of 10× or more and are an active topic of experimentation in both competitive and recreational esports communities. Inspired by work in pointer-based gain optimization and extending our previous results from the first user study focused on mouse sensitivity in first-person targeting tasks (Boudaoud et al., 2023), we describe a range of optimal mouse sensitivity wherein players perform statistically significantly better in task completion time and throughput. For tasks involving first-person view control, mouse sensitivity is best described using the ratio between an in-game rotation of the view and corresponding physical displacement of the mouse. We discuss how this displacement-to-rotation sensitivity is incompatible with the control-display gain reported in traditional pointer-based gain studies as well as other rotational gains reported in head-controlled interface studies. We provide additional details regarding impacts of mouse dots per inch, on reported sensitivity, the distribution of spatial difficulty in our experiment, our submovement parsing algorithm, and relationships between measured parameters, further demonstrating optimal sensitivity arising from a speed-precision tradeoff. We conclude our work by updating and improving our suggestions for mouse sensitivity selection and refining directions for future work.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"15 4","pages":"493-506"},"PeriodicalIF":2.3,"publicationDate":"2023-07-17","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"138678654","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Subjective and Objective Analysis of Streamed Gaming Videos","authors":"Xiangxu Yu;Zhenqiang Ying;Neil Birkbeck;Yilin Wang;Balu Adsumilli;Alan C. Bovik","doi":"10.1109/TG.2023.3293093","DOIUrl":"10.1109/TG.2023.3293093","url":null,"abstract":"The rising popularity of online user-generated-content (UGC) in the form of streamed and shared videos has hastened the development of perceptual video quality assessment (VQA) models, which can be used to help optimize their delivery. Gaming videos, which are a relatively new type of UGC videos, are created when skilled and casual gamers post videos of their gameplay. These kinds of screenshots of UGC gameplay videos have become extremely popular on major streaming platforms, such as YouTube and Twitch. Synthetically generated gaming content presents challenges to existing VQA algorithms, including those based on natural scene/video statistics models. Synthetically generated gaming content presents different statistical behavior than naturalistic videos. A number of studies have been directed toward understanding the perceptual characteristics of professionally generated gaming videos arising in gaming video streaming, online gaming, and cloud gaming. However, little work has been done on understanding the quality of UGC gaming videos, and how it can be characterized and predicted. Toward boosting the progress of gaming video VQA model development, we conducted a comprehensive study of subjective and objective VQA models on UGC gaming videos. To do this, we created a novel UGC gaming video resource, called the LIVE-YouTube Gaming video quality (LIVE-YT-Gaming) database, comprised of 600 real UGC gaming videos. We conducted a subjective human study on this data, yielding 18 600 human quality ratings recorded by 61 human subjects. We also evaluated a number of state-of-the-art VQA models on the new database, including a new one, called GAME-VQP, based on both natural video statistics and CNN-learned features. To help support work in this field, we are making the new LIVE-YT-Gaming Database, along with code for GAME-VQP, publicly available through the link: \u0000<uri>https://live.ece.utexas.edu/research/LIVE-YT-Gaming/index.html</uri>\u0000.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 2","pages":"445-458"},"PeriodicalIF":2.3,"publicationDate":"2023-07-07","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"135182601","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Modeling Game Mechanics With Ceptre","authors":"Chris Martens;Alexander Card;Henry Crain;Asha Khatri","doi":"10.1109/TG.2023.3292982","DOIUrl":"10.1109/TG.2023.3292982","url":null,"abstract":"Game description languages have a variety of uses, including formal reasoning about the emergent consequences of a game's mechanics, implementation of artificial intelligence decision making where the game's rules make up the space of possible actions, automated game and level generation, and game prototyping for the sake of low-time-investment design and tinkering. However, in practice, a new game description language has been invented for almost every new use case, without providing formal underpinnings that follow generalizable principles and can be reasoned about separately from the specific software implementation of the language. Ceptre is a language that attempts to break this pattern, based on an old idea known as multiset rewriting. This article describes the language formally, through example, and in a tutorial style, then demonstrates its use for writing formal specifications of game mechanics so that they may be interactively explored, queried, and analyzed in a computational framework. Ceptre allows designers to step through executions, interact with the mechanics from the standpoint of a player, run random simulated playthroughs, collect and analyze data from said playthroughs, and formally verify mathematical properties of the mechanics, and it has been used in a number of research projects since its inception, for applications such as procedural narrative generation, formal game modeling, and game AI.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 2","pages":"431-444"},"PeriodicalIF":2.3,"publicationDate":"2023-07-06","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"62570270","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Image Augmentation-Based Momentum Memory Intrinsic Reward for Sparse Reward Visual Scenes","authors":"Zheng Fang;Biao Zhao;Guizhong Liu","doi":"10.1109/TG.2023.3288042","DOIUrl":"https://doi.org/10.1109/TG.2023.3288042","url":null,"abstract":"Many real-life tasks can be abstracted as sparse reward visual scenes, which can make it difficult for an agent to accomplish tasks accepting only images and sparse reward. To address this problem, we split it into two parts: visual representation and sparse reward, and propose our novel framework, called image augmentation-based momentum memory intrinsic reward, which combines self-supervised representation learning with intrinsic motivation. For visual representation, we acquire a representation driven by a combination of image-augmented forward dynamics and reward. To handle sparse reward, we design a new type of intrinsic reward called momentum memory intrinsic reward, which uses the difference between the outputs from the current model (online network) and the historical model (target network) to indicate the agent's state familiarity. We evaluate our method on a visual navigation task with sparse reward in VizDoom and demonstrate that it achieves state-of-the-art performance in terms of sample efficiency. Our method is at least two times faster than existing methods and reaches a 100% success rate.","PeriodicalId":55977,"journal":{"name":"IEEE Transactions on Games","volume":"16 3","pages":"509-517"},"PeriodicalIF":1.7,"publicationDate":"2023-06-20","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"142235806","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":4,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}