IEEE Transactions on Multimedia最新文献

{"title":"Language-Inspired Unknown-Space Expansion for Few-Shot Open-Set Detection","authors":"Zhimeng Xin;Tianxu Wu;Yixiong Zou;Shiming Chen;Qinmu Peng;Xinge You","doi":"10.1109/TMM.2026.3673530","DOIUrl":"https://doi.org/10.1109/TMM.2026.3673530","url":null,"abstract":"<bold>F</b>ew-shot <bold>o</b>pen-set <bold>o</b>bject <bold>d</b>etection (FOOD) aims to effectively detect known objects with a limited number of annotated samples while rejecting unknown ones. Due to the lack of real unknown data during training, existing methods treat high-uncertainty samples, e.g., background objects of known images, as simulated unknown objects for model optimization. However, such simulated unknown data can hardly reflect the distribution of real unknown data due to its similarity to the known data, and the limited number of known data even exacerbates this problem. To tackle this issue, we are inspired by large language models (LLM) for their rich prior knowledge to expand the simulated unknown data. We propose the Unknown Feature Space Expansion (UFSE) module to take simulated unknown data as a word that describes the rational unknown space for the given known data, and transforms them into inputs for a common text encoder, enriching the simulated unknown features. Following this, we develop an Unknown Pseudo-Classifier (UPC) as another component of UFSE to effectively identify the expanded unknown features from the background, further increasing the difference between unknown and known samples and improving models’ ability to distinguish real unknown data. Extensive experiments on the VOC and COCO datasets show that our proposed UFSE module achieves the state-of-the-art performance of FOOD, with 14.6% improvement in the mean average precision of novel classes in the VOC 1-shot setting.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"7697-7708"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149034364","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"TunCLIP: Adaptive Multi-Scale Zero-Shot Anomaly Detection via Vision-Language Feature Infiltration and Semantic Enhancement","authors":"Tao Zhang;Wenbo Pang;Xuequan Lu","doi":"10.1109/TMM.2026.3684998","DOIUrl":"https://doi.org/10.1109/TMM.2026.3684998","url":null,"abstract":"Zero-shot anomaly detection (ZSAD) is critical for identifying unseen anomalies without prior abnormal samples, particularly in safety-critical domains such as industrial inspection and medical diagnostics. While Vision-Language Models (VLMs) like CLIP show significant promise, existing CLIP-based ZSAD methods often struggle with adaptive anomaly region exploration, effective multi-scale feature fusion, and the extraction of subtle semantic anomalies. To address these limitations, this paper introduces TunCLIP, a novel and unified framework designed to enhance ZSAD performance through vision-language feature infiltration and semantic augmentation. TunCLIP integrates three complementary core modules: 1) Adaptive Anomaly-Probing Attentive Feature Infiltration (AAFI), which dynamically highlights and amplifies potential anomalous regions by generating spatial attention masks and injecting learnable tokens; 2) Hierarchical Multi-scale Latent Anomaly Spectrum Decoder (H-LASD), which captures cross-scale anomaly signals by modeling residual deviations against estimated normal baselines across multi-layer features; and 3) Anomaly Semantic Enhancement (ASE), which amplifies weak anomaly semantics by inverting channel affinity relationships to emphasize less cohesive, anomaly-indicative patterns. Extensive experiments on diverse industrial benchmarks (MVTec-AD, VisA, BTAD, DTD-Synthetic) and medical datasets (HeadCT, BrainMRI, ColonDB, TN3k, BMAD) demonstrate TunCLIP's superior performance, achieving average image-level AUROC of 90.2% and pixel-level AUROC of 95.8% on industrial datasets, and 95.6% and 87.3% respectively on medical datasets, consistently outperforming state-of-the-art methods. This research provides a robust, interpretable, and highly effective solution for ZSAD in complex real-world multimedia environments.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"7568-7578"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149034371","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"MF-GGNN: Crack Visual Reasoning CAPTCHA Holistically Using a Novel Multi-Feature Fusion-Based Graph Gated Neural Network","authors":"Botao Xu;Haizhou Wang;Wenxian Wang;Xingshu Chen","doi":"10.1109/TMM.2025.3645614","DOIUrl":"https://doi.org/10.1109/TMM.2025.3645614","url":null,"abstract":"With the advancement of computer vision and deep learning algorithms, many traditional text-based and image-based CAPTCHAs (Completely Automated Public Turing test to tell Computers and Humans Apart) have faced the significant risk of being automatically recognized and cracked. To enhance security, some leading CAPTCHA service providers have begun to explore interactive CAPTCHAs that integrate visual reasoning tasks, which greatly increase machine recognition difficulty. Compared to traditional CAPTCHAs, these CAPTCHAs leverage the unique advantage of the human brain in comprehensively analyzing and judging real-world information. Existing visual reasoning CAPTCHA research employs non-end-to-end model. However, it is unable to fully learn the rich features from CAPTCHAs through holistically modeling the logical relations. Additionally, the modular approach requires training modules separately and assembling them at the end, leading to substantial manual effort. To address the issues mentioned above, this paper presents an end-to-end learning framework called MF-GGNN (Multi-Feature Fusion-based Graph Gated Neural Network). The framework utilizes GGNN to simulate human association and reasoning over visual content. Specifically, we first locate objects in the image using an object detection model. We then extract multiple features and generate question attention distribution. Finally, we integrate the above outputs using a graph neural gated network for multi-step reasoning. Our model can capture the deep intrinsic connections between objects and their spatial information to achieve end-to-end graph reasoning. MF-GGNN achieves an average attack success rate of 92.2% across multiple CAPTCHAs. The results of various experiments demonstrate that our method can excellently accomplish visual reasoning tasks in CAPTCHAs. In addition, we construct a large-scale, open-source dataset, namely ViRC (<underline>Vi</u>sual <underline>R</u>easoning <underline>C</u>APTCHAs), including over 50,000 CAPTCHAs and corresponding questions. Our research findings will provide references and insights for evaluating and designing more secure and robust next-generation CAPTCHAs.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"1790-1805"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149034346","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"CHFusion: A Cross-Modality High-Resolution Representation Framework for Infrared and Visible Image Fusion","authors":"Biaojian Jin;Rencan Nie;Jinde Cao;Ying Zhang;Dongyang Li","doi":"10.1109/TMM.2023.3294814","DOIUrl":"10.1109/TMM.2023.3294814","url":null,"abstract":"In our study, we proposed a novel infrared and visible image fusion framework based on cross-modality transfer and high-resolution representation, termed as CHFusion. On the one hand, the high-resolution representation backbone is devised to receive multi-scale information and maintain high-resolution representation. More specifically, the proposed method involves the pyramid cross-modality feature transfer module to achieve information interaction with different modalities and resolutions. In particular, we introduce the gradient block to obtain texture information of the source image and then supplement it with high-resolution features. We utilize the adaptive channel attention block to compress high-resolution features and then guide image reconstruction. Moreover, a cross-modality high-resolution aggregation block is used to integrate multi-scale information. On the other hand, we propose a difference-aware algorithm, which can generate a pair of weights and then use the weights to construct the difference-aware loss, and then difference-aware loss, a texture loss, and an intensity loss to guide our network to preserve abundant texture information and optimally salient target. Extensive experiments demonstrate the superiority of our method over state-of-the-art alternatives in terms of object maintenance and texture preservation.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"8784-8797"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"63097064","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"A Circular Window-Based Cascade Transformer for Online Action Detection","authors":"Shuqiang Cao;Weixin Luo;Bairui Wang;Ran Song;Wei Zhang;Lin Ma","doi":"10.1109/TMM.2026.3676664","DOIUrl":"https://doi.org/10.1109/TMM.2026.3676664","url":null,"abstract":"Online Action Detection (OAD) aims to identify ongoing actions in untrimmed video streams, which is critical for real-world applications. However, effectively modeling long-term temporal dependencies while maintaining real-time inference efficiency remains challenging. To address this challenge, we propose a circular window-based cascade Transformer (CWC-Trans), featuring three key components: (i) a window-based hierarchical architecture that captures fine-grained temporal dynamics, and a circular window updating mechanism that enables efficient long-term history modeling; (ii) an auxiliary offline action segmentation (OAS) task that guides the hierarchical architecture to learn discriminative long-term action patterns while suppressing background interference; and (iii) a cascade prediction refinement module with adaptive kernels that progressively refines action predictions. By sharing the classifier and refinement modules between OAD and OAS tasks, CWC-Trans integrates multi-level long- and short-term information across feature, probabilistic, and refined probabilistic levels. Extensive experiments on THUMOS’14, TVSeries, and HDD demonstrate that CWC-Trans achieves state-of-the-art performance with high inference efficiency.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"8151-8163"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149130726","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Learning From the Abyss: Self-Adaptive Distillation for Enhancing Underwater Images With Complex Degradations","authors":"Yu Zhao;Xiaofeng Cong;Chuanrui Hu;Lu Dong;Jie Gui;James Tin-Yau Kwok;Yuyan Yan Tang","doi":"10.1109/TMM.2026.3689437","DOIUrl":"https://doi.org/10.1109/TMM.2026.3689437","url":null,"abstract":"While deep learning has advanced underwater image enhancement (UIE), progress is fundamentally hampered by the scarcity of paired training data. Semi-supervised methods, which leverage abundant unlabeled images, offer a promising solution but often employ naive, static weighting for pseudo-labels. This overlooks a key principle: the most informative training samples are those that represent a successful restoration from severe degradation, thus possessing a greater transformation difficulty. To address this, we propose a novel adaptive training paradigm, the Self-Distilled Degradation-Aware Network (SDDA-Net), governed by an incentive-driven weighting mechanism. We introduce a scoring function that evaluates each pseudo-label by considering both its perceptual quality and a term that is inversely proportional to its structural similarity to the input. This score, which quantifies the difficulty and success of the enhancement, is then used to dynamically modulate the sample’s influence on the unsupervised loss. By allocating greater learning weights to successfully restored, complex examples, our framework forces the model to focus on mastering challenging degradations. Extensive experiments on multiple benchmark datasets demonstrate that this adaptive approach significantly outperforms current state-of-the-art methods, validating that the strategic, sample-adaptive modulation of the learning process is critical for success in data-scarce restoration tasks.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"9269-9280"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149130693","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Hierarchical Structure Consistency Learning for Multimodal Emotion Recognition","authors":"Boan Yang;Qinghongya Shi;Xiaofen Zong;Mang Ye","doi":"10.1109/TMM.2026.3694526","DOIUrl":"https://doi.org/10.1109/TMM.2026.3694526","url":null,"abstract":"Multimodal Emotion Recognition (MER) aims to infer emotions from multimodal data. Existing studies have leveraged attention mechanisms and modal fusion strategies with significant success in emotional understanding. However, these methods struggle with inadequate intra-modal interaction and modal asynchrony. This study introduces an innovative Hierarchical Structure Consistency Learning method for Multimodal Emotion Recognition (HSCL) to tackle these issues. As for inadequate intra-modal interaction, HSCL designs a semantic consistency algorithm with a margin constraint to enhance modal aggregation. This approach focuses on strengthening intra-modal interactions and addressing the inadequacies in emotional context comprehension, thereby deepening the understanding within each modality. For modal asynchrony, HSCL optimizes representation consistency at the modality level and geometric consistency at the sample level, addressing modal asynchrony and improving model performance. In addition to applying our structure consistency on the original multimodal features, our HSCL computes reconstructed features from decoupled representations and performs hierarchical consistency calculations on these reconstructed features. This dual application mitigates the potential negative impact of the training process on consistency, thereby enhancing adaptability and performance.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"9623-9637"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149130832","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"Photovoltaic Defect Image Generator With Boundary Alignment Smoothing Constraint for Domain Shift Mitigation","authors":"Dongying Li;Binyi Su;Hua Zhang;Yong Li;Haiyong Chen","doi":"10.1109/TMM.2026.3679929","DOIUrl":"https://doi.org/10.1109/TMM.2026.3679929","url":null,"abstract":"Accurate defect detection of photovoltaic (PV) cells is critical for ensuring quality and efficiency in intelligent PV manufacturing systems. However, the scarcity of rich defect data poses substantial challenges for effective model training. While existing methods have explored generative models to augment datasets, they often suffer from instability, limited diversity, and domain shifts. To address these issues, we propose PDIG, a <underline>P</u>hotovoltaic <underline>D</u>efect <underline>I</u>mage <underline>G</u>enerator based on Stable Diffusion (SD). PDIG leverages the strong priors learned from large-scale datasets to enhance generation quality under limited data. Specifically, we introduce a <underline>S</u>emantic <underline>C</u>oncept <underline>E</u>mbedding (SCE) module that incorporates text-conditioned priors to capture the relational concepts between defect types and their appearances. To further enrich the domain distribution, we design a <underline>L</u>ightweight <underline>I</u>ndustrial <underline>S</u>tyle <underline>A</u>daptor (LISA), which injects industrial defect characteristics into the SD model through cross-disentangled attention. At inference, we propose a <underline>T</u>ext-<underline>I</u>mage <underline>D</u>ual-<underline>S</u>pace <underline>C</u>onstraints (TIDSC) module, enforcing the quality of generated images via positional consistency and spatial smoothing alignment. Extensive experiments demonstrate that PDIG achieves superior realism and diversity compared to state-of-the-art methods. Specifically, our approach improves Frechet Inception Distance (FID) by 19.16 points over the second-best method and significantly enhances the performance of downstream defect detection tasks.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"8481-8493"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149130899","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"RCOC-Distill: Boosting 4D Radar-Camera Online Calibration With Knowledge Distillation From LiDAR Features","authors":"Long Zhuang;Yiqing Yao;Tao Zhang","doi":"10.1109/TMM.2026.3673505","DOIUrl":"https://doi.org/10.1109/TMM.2026.3673505","url":null,"abstract":"Extrinsic calibration of 4D millimeter-wave radar and camera is a prerequisite for multi-sensor fusion in complex environments. Online calibration algorithms can dynamically correct jitter errors caused by vibrations and have increasingly become the predominant approach. However, online calibration for 4D radar-camera systems is constrained by the sparsity of radar point clouds, making it challenging to establish feature correspondences between radar and camera. This paper proposes a novel 4D radar-camera online calibration algorithm, RCOC-Distill, which leverages cross-modal knowledge distillation (CMKD) to enhance 4D radar representations with LiDAR-derived features, thereby improving calibration accuracy. A Local Correlation Layer (LCL) is introduced to compute feature similarity within a local window via convolution operations, reducing computational burden while maintaining local consistency. Additionally, a Radar-LiDAR Cross-Modal Alignment (RLCMA) module is proposed to further enhance radar feature density, mitigating inefficiencies in knowledge transfer from LiDAR to 4D radar. The proposed method is evaluated on two datasets (NTU4DRaDLM and Dual-Radar). Compared to state-of-the-art models, RCOC-Distill achieves improvements of 38.73% and 23.48% in translation and rotation errors, respectively, on the Dual-Radar dataset. These results validate the superiority of the proposed method.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"7936-7949"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149131171","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}

{"title":"SANR: Scene-Aware Neural Representation for Light Field Image Compression With Rate-Distortion Optimization","authors":"Gai Zhang;Xinfeng Zhang;Lv Tang;Hongyu An;Li Zhang;Qingming Huang","doi":"10.1109/TMM.2026.3686627","DOIUrl":"https://doi.org/10.1109/TMM.2026.3686627","url":null,"abstract":"Light field images capture multi-view scene information and play a crucial role in 3D scene reconstruction. However, their high-dimensional nature results in enormous data volumes, posing a significant challenge for efficient compression in practical storage and transmission scenarios. Although neural representation-based methods have shown promise in light field image compression, most approaches rely on direct coordinate-to-pixel mapping through implicit neural representation (INR), often neglecting the explicit modeling of scene structure. Moreover, they typically lack end-to-end rate-distortion optimization, limiting their compression efficiency. To address these limitations, we propose SANR, a Scene-Aware Neural Representation framework for light field image compression with end-to-end rate-distortion optimization. For scene awareness, SANR introduces a hierarchical scene modeling block that leverages multi-scale latent codes to capture intrinsic scene structures, thereby reducing the information gap between INR input coordinates and the target light field image. From a compression perspective, SANR is the first to incorporate entropy-constrained quantization-aware training (QAT) into neural representation-based light field image compression, enabling end-to-end rate-distortion optimization. Extensive experiment results demonstrate that SANR significantly outperforms state-of-the-art techniques regarding rate-distortion performance with a 65.62% BD-rate saving against HEVC.","PeriodicalId":13273,"journal":{"name":"IEEE Transactions on Multimedia","volume":"28 ","pages":"9321-9332"},"PeriodicalIF":9.9,"publicationDate":"2026-01-01","publicationTypes":"Journal Article","fieldsOfStudy":null,"isOpenAccess":false,"openAccessPdf":"","citationCount":null,"resultStr":null,"platform":"Semanticscholar","paperid":"149131217","PeriodicalName":null,"FirstCategoryId":null,"ListUrlMain":null,"RegionNum":1,"RegionCategory":"计算机科学","ArticlePicture":[],"TitleCN":null,"AbstractTextCN":null,"PMCID":"","EPubDate":null,"PubModel":null,"JCR":null,"JCRName":null,"Score":null,"Total":0}