Java多模态案例

我将为您创建一个完整的Java多模态处理案例,展示如何同时处理文本、图像和音频数据。

完整的Java多模态处理案例

项目结构

multimodal-demo/
├── pom.xml
└── src/
    └── main/
        └── java/
            └── com/example/multimodal/
                ├── MultimodalApplication.java
                ├── core/
                │   ├── ModalityProcessor.java
                │   ├── TextProcessor.java
                │   ├── ImageProcessor.java
                │   └── AudioProcessor.java
                ├── fusion/
                │   └── MultimodalFusionEngine.java
                ├── model/
                │   ├── MultimodalData.java
                │   └── FusionResult.java
                └── util/
                    ├── ImageUtils.java
                    └── AudioUtils.java

Maven依赖 (pom.xml)

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
         http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.example</groupId>
    <artifactId>multimodal-demo</artifactId>
    <version>1.0-SNAPSHOT</version>
    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
    <dependencies>
        <!-- OpenNLP for text processing -->
        <dependency>
            <groupId>org.apache.opennlp</groupId>
            <artifactId>opennlp-tools</artifactId>
            <version>2.1.0</version>
        </dependency>
        <!-- OpenCV for image processing -->
        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>javacv-platform</artifactId>
            <version>1.5.9</version>
        </dependency>
        <!-- Java Sound API for audio -->
        <dependency>
            <groupId>com.googlecode.soundlibs</groupId>
            <artifactId>tarsosdsp</artifactId>
            <version>2.4</version>
        </dependency>
        <!-- JSON processing -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.15.2</version>
        </dependency>
        <!-- SLF4J for logging -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>2.0.7</version>
        </dependency>
    </dependencies>
</project>

核心模型类

package com.example.multimodal.model;
import java.util.HashMap;
import java.util.Map;
public class MultimodalData {
    public enum ModalityType {
        TEXT, IMAGE, AUDIO
    }
    private String text;
    private byte[] imageData;
    private byte[] audioData;
    private Map<String, Object> metadata = new HashMap<>();
    // Getters and Setters
    public String getText() {
        return text;
    }
    public void setText(String text) {
        this.text = text;
        if (text != null) {
            metadata.put("text_length", text.length());
        }
    }
    public byte[] getImageData() {
        return imageData;
    }
    public void setImageData(byte[] imageData) {
        this.imageData = imageData;
        if (imageData != null) {
            metadata.put("image_size", imageData.length);
        }
    }
    public byte[] getAudioData() {
        return audioData;
    }
    public void setAudioData(byte[] audioData) {
        this.audioData = audioData;
        if (audioData != null) {
            metadata.put("audio_size", audioData.length);
        }
    }
    public void addMetadata(String key, Object value) {
        metadata.put(key, value);
    }
    public Object getMetadata(String key) {
        return metadata.get(key);
    }
    public Map<String, Object> getAllMetadata() {
        return metadata;
    }
    public boolean hasText() {
        return text != null && !text.isEmpty();
    }
    public boolean hasImage() {
        return imageData != null && imageData.length > 0;
    }
    public boolean hasAudio() {
        return audioData != null && audioData.length > 0;
    }
}

package com.example.multimodal.model;
import java.util.HashMap;
import java.util.Map;
public class FusionResult {
    private Map<String, Object> features = new HashMap<>();
    private Map<String, Object> confidence = new HashMap<>();
    private String dominantModality;
    private String overallSentiment;
    private double overallScore;
    public void addFeature(String key, Object value) {
        features.put(key, value);
    }
    public void addConfidence(String modality, Double value) {
        confidence.put(modality, value);
    }
    // Getters and Setters
    public Map<String, Object> getFeatures() {
        return features;
    }
    public void setFeatures(Map<String, Object> features) {
        this.features = features;
    }
    public Map<String, Object> getConfidence() {
        return confidence;
    }
    public void setConfidence(Map<String, Object> confidence) {
        this.confidence = confidence;
    }
    public String getDominantModality() {
        return dominantModality;
    }
    public void setDominantModality(String dominantModality) {
        this.dominantModality = dominantModality;
    }
    public String getOverallSentiment() {
        return overallSentiment;
    }
    public void setOverallSentiment(String overallSentiment) {
        this.overallSentiment = overallSentiment;
    }
    public double getOverallScore() {
        return overallScore;
    }
    public void setOverallScore(double overallScore) {
        this.overallScore = overallScore;
    }
}

核心处理器

package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
public interface ModalityProcessor {
    String getModalityType();
    Object process(MultimodalData data);
}

package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
public class TextProcessor implements ModalityProcessor {
    private MatrixOperations matrixOps;
    public TextProcessor() {
        this.matrixOps = new MatrixOperations();
    }
    @Override
    public String getModalityType() {
        return "TEXT";
    }
    @Override
    public Map<String, Object> process(MultimodalData data) {
        if (!data.hasText()) {
            return null;
        }
        String text = data.getText();
        Map<String, Object> textFeatures = new HashMap<>();
        // 1. 文本统计特征
        textFeatures.put("word_count", countWords(text));
        textFeatures.put("char_count", text.length());
        textFeatures.put("sentence_count", countSentences(text));
        textFeatures.put("unique_words", getUniqueWords(text));
        // 2. 情感分析（简化版）
        textFeatures.put("sentiment_score", analyzeSentiment(text));
        textFeatures.put("sentiment_category", getSentimentCategory(
            (double) textFeatures.get("sentiment_score")));
        // 3. 关键词提取（简化版）
        textFeatures.put("keywords", extractKeywords(text));
        // 4. 文本向量化（模拟Word2Vec）
        double[] textVector = vectorizeText(text);
        textFeatures.put("vector", textVector);
        data.addMetadata("text_features", textFeatures);
        data.addMetadata("text_vector", textVector);
        return textFeatures;
    }
    private int countWords(String text) {
        return text.trim().split("\\s+").length;
    }
    private int countSentences(String text) {
        return text.split("[.!?]").length;
    }
    private int getUniqueWords(String text) {
        return (int) Arrays.stream(text.toLowerCase().split("\\s+"))
                          .distinct()
                          .count();
    }
    private double analyzeSentiment(String text) {
        String[] positiveWords = {"good", "great", "excellent", "wonderful", "amazing", 
                                  "happy", "love", "like", "beautiful", "awesome"};
        String[] negativeWords = {"bad", "terrible", "awful", "horrible", "hate", 
                                 "ugly", "poor", "worst", "worried", "sad"};
        String[] words = text.toLowerCase().split("\\s+");
        int positiveCount = 0;
        int negativeCount = 0;
        for (String word : words) {
            if (Arrays.asList(positiveWords).contains(word)) {
                positiveCount++;
            }
            if (Arrays.asList(negativeWords).contains(word)) {
                negativeCount++;
            }
        }
        return (positiveCount - negativeCount) / (double) words.length;
    }
    private String getSentimentCategory(double score) {
        if (score > 0.1) return "Positive";
        if (score < -0.1) return "Negative";
        return "Neutral";
    }
    private String[] extractKeywords(String text) {
        // 简化提取：过滤停用词后取高频词
        String[] stopWords = {"the", "a", "an", "and", "or", "but", "is", "are", 
                             "was", "were", "be", "been", "being"};
        return Arrays.stream(text.toLowerCase().split("\\s+"))
                    .filter(word -> !Arrays.asList(stopWords).contains(word))
                    .distinct()
                    .limit(10)
                    .toArray(String[]::new);
    }
    private double[] vectorizeText(String text) {
        // 模拟文本向量化，实际应用中会使用预训练模型
        String[] words = text.toLowerCase().split("\\s+");
        double[] vector = new double[100];
        for (String word : words) {
            // 使用哈希函数模拟词嵌入
            int hashCode = Math.abs(word.hashCode());
            int index = hashCode % vector.length;
            vector[index] += 1.0;
        }
        // 归一化
        matrixOps.normalizeVector(vector);
        return vector;
    }
}

图像处理器

package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
import com.example.multimodal.util.ImageUtils;
import org.bytedeco.javacv.Java2DFrameConverter;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.bytedeco.opencv.opencv_core.Mat;
import org.bytedeco.opencv.opencv_core.Scalar;
import org.bytedeco.opencv.opencv_imgproc.Vec2d;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Map;
public class ImageProcessor implements ModalityProcessor {
    private ImageUtils imageUtils;
    public ImageProcessor() {
        this.imageUtils = new ImageUtils();
    }
    @Override
    public String getModalityType() {
        return "IMAGE";
    }
    @Override
    public Map<String, Object> process(MultimodalData data) {
        if (!data.hasImage()) {
            return null;
        }
        try {
            byte[] imageBytes = data.getImageData();
            BufferedImage image = ImageIO.read(new ByteArrayInputStream(imageBytes));
            if (image == null) {
                return null;
            }
            Map<String, Object> imageFeatures = new HashMap<>();
            // 1. 基本图像特征
            imageFeatures.put("width", image.getWidth());
            imageFeatures.put("height", image.getHeight());
            imageFeatures.put("color_count", countColors(image));
            imageFeatures.put("brightness", calculateAverageBrightness(image));
            // 2. 颜色特征
            double[] colorHistogram = extractColorHistogram(image);
            imageFeatures.put("color_histogram", colorHistogram);
            // 3. 纹理特征（简化版）
            Map<String, Object> textureFeatures = extractTextureFeatures(image);
            imageFeatures.put("texture", textureFeatures);
            // 4. 图像向量化
            double[] imageVector = vectorizeImage(image);
            imageFeatures.put("vector", imageVector);
            data.addMetadata("image_features", imageFeatures);
            data.addMetadata("image_vector", imageVector);
            return imageFeatures;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }
    private int countColors(BufferedImage image) {
        java.util.Set<Integer> colors = new java.util.HashSet<>();
        int stride = 5; // 采样以提高性能
        for (int x = 0; x < image.getWidth(); x += stride) {
            for (int y = 0; y < image.getHeight(); y += stride) {
                colors.add(image.getRGB(x, y));
            }
        }
        return colors.size();
    }
    private double calculateAverageBrightness(BufferedImage image) {
        long sum = 0;
        int count = 0;
        int stride = 5;
        for (int x = 0; x < image.getWidth(); x += stride) {
            for (int y = 0; y < image.getHeight(); y += stride) {
                java.awt.Color c = new java.awt.Color(image.getRGB(x, y));
                sum += (c.getRed() + c.getGreen() + c.getBlue()) / 3;
                count++;
            }
        }
        return sum / (double) count;
    }
    private double[] extractColorHistogram(BufferedImage image) {
        double[] histogram = new double[64]; // 使用量化颜色空间
        int bucketsPerChannel = 4; // 4^3 = 64 buckets
        int stride = 3;
        for (int x = 0; x < image.getWidth(); x += stride) {
            for (int y = 0; y < image.getHeight(); y += stride) {
                java.awt.Color c = new java.awt.Color(image.getRGB(x, y));
                int r = c.getRed() / 64; // 4 buckets for red
                int g = c.getGreen() / 64;
                int b = c.getBlue() / 64;
                int index = (r * bucketsPerChannel * bucketsPerChannel) + 
                            (g * bucketsPerChannel) + b;
                histogram[index]++;
            }
        }
        // 归一化直方图
        double sum = 0;
        for (double value : histogram) sum += value;
        if (sum > 0) {
            for (int i = 0; i < histogram.length; i++) {
                histogram[i] /= sum;
            }
        }
        return histogram;
    }
    private Map<String, Object> extractTextureFeatures(BufferedImage image) {
        Map<String, Object> features = new HashMap<>();
        // 计算灰度共生矩阵（简化的统计特征）
        double[] grayLevels = new double[256];
        int count = 0;
        int stride = 5;
        for (int x = 0; x < image.getWidth(); x += stride) {
            for (int y = 0; y < image.getHeight(); y += stride) {
                java.awt.Color c = new java.awt.Color(image.getRGB(x, y));
                int gray = (int) (0.299 * c.getRed() + 
                                0.587 * c.getGreen() + 
                                0.114 * c.getBlue());
                grayLevels[gray]++;
                count++;
            }
        }
        // 计算统计特征
        double mean = 0;
        double stdDev = 0;
        for (int i = 0; i < 256; i++) {
            double probability = grayLevels[i] / count;
            mean += i * probability;
        }
        for (int i = 0; i < 256; i++) {
            double probability = grayLevels[i] / count;
            stdDev += (i - mean) * (i - mean) * probability;
        }
        stdDev = Math.sqrt(stdDev);
        features.put("mean_gray", mean);
        features.put("std_dev", stdDev);
        features.put("smoothness", 1.0 / (1.0 + stdDev * stdDev));
        // 能量和熵
        double energy = 0;
        double entropy = 0;
        for (int i = 0; i < 256; i++) {
            double probability = grayLevels[i] / count;
            if (probability > 0) {
                energy += probability * probability;
                entropy -= probability * (Math.log(probability) / Math.log(2));
            }
        }
        features.put("energy", energy);
        features.put("entropy", entropy);
        return features;
    }
    private double[] vectorizeImage(BufferedImage image) {
        // 转换图像为特征向量（使用颜色直方图和空间特征）
        double[] colorHist = extractColorHistogram(image);
        Map<String, Object> texture = extractTextureFeatures(image);
        // 组合特征
        double[] vector = new double[70];
        System.arraycopy(colorHist, 0, vector, 0, colorHist.length);
        vector[64] = (Double) texture.get("mean_gray");
        vector[65] = (Double) texture.get("std_dev");
        vector[66] = (Double) texture.get("smoothness");
        vector[67] = (Double) texture.get("energy");
        vector[68] = (Double) texture.get("entropy");
        vector[69] = (Double) texture.get("entropy");
        // 归一化
        double sum = 0;
        for (double v : vector) sum += v * v;
        if (sum > 0) {
            double scale = Math.sqrt(sum);
            for (int i = 0; i < vector.length; i++) vector[i] /= scale;
        }
        return vector;
    }
}

音频处理器

package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
import javax.sound.sampled.*;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Map;
public class AudioProcessor implements ModalityProcessor {
    @Override
    public String getModalityType() {
        return "AUDIO";
    }
    @Override
    public Map<String, Object> process(MultimodalData data) {
        if (!data.hasAudio()) {
            return null;
        }
        try {
            byte[] audioBytes = data.getAudioData();
            Map<String, Object> audioFeatures = new HashMap<>();
            // 1. 基本信息
            audioFeatures.put("duration", calculateDuration(audioBytes));
            audioFeatures.put("sample_rate", 44100); // 典型采样率
            audioFeatures.put("channels", 2);
            // 2. 波形特征
            AudioWaveform waveform = extractWaveform(audioBytes);
            audioFeatures.put("amplitude", waveform.getAmplitudeFeatures());
            audioFeatures.put("rms", waveform.getRMS());
            // 3. 频域特征
            Map<String, Object> frequencyFeatures = extractFrequencyFeatures(audioBytes);
            audioFeatures.put("frequency", frequencyFeatures);
            // 4. 音频向量化
            double[] audioVector = vectorizeAudio(audioFeatures);
            audioFeatures.put("vector", audioVector);
            data.addMetadata("audio_features", audioFeatures);
            data.addMetadata("audio_vector", audioVector);
            return audioFeatures;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }
    private double calculateDuration(byte[] audioBytes) {
        // 假设：16位PCM，44100Hz，双声道 -> Bytes = samples * 2 * 2
        int sampleSize = 2; // 16位
        int channels = 2;
        int sampleRate = 44100;
        long totalSamples = audioBytes.length / (sampleSize * channels);
        return totalSamples / (double) sampleRate;
    }
    private AudioWaveform extractWaveform(byte[] audioBytes) {
        AudioWaveform waveform = new AudioWaveform();
        int samples = audioBytes.length / 2;
        double[] amplitudes = new double[samples / 100]; // 降采样
        double maxAmplitude = 0;
        double sumSquares = 0;
        int count = 0;
        for (int i = 0; i < samples; i += 100) {
            int startSample = i * 2;
            if (startSample + 1 < audioBytes.length) {
                short sample = (short) (((audioBytes[startSample] & 0xFF) << 8) |
                                       (audioBytes[startSample + 1] & 0xFF));
                double amplitude = sample / 32768.0;
                if (count < amplitudes.length) {
                    amplitudes[count] = amplitude;
                    count++;
                }
                if (Math.abs(amplitude) > maxAmplitude) {
                    maxAmplitude = Math.abs(amplitude);
                }
                sumSquares += amplitude * amplitude;
            }
        }
        waveform.setAmplitude(amplitudes);
        waveform.setMaxAmplitude(maxAmplitude);
        waveform.setRMS(Math.sqrt(sumSquares / samples));
        return waveform;
    }
    private Map<String, Object> extractFrequencyFeatures(byte[] audioBytes) {
        Map<String, Object> features = new HashMap<>();
        // 简化的FFT分析（计算过零率等）
        int zeroCrossings = countZeroCrossings(audioBytes);
        features.put("zero_crossing_rate", zeroCrossings);
        double[] frequencyBands = analyzeFrequencyBands(audioBytes);
        features.put("low_band", frequencyBands[0]);
        features.put("mid_band", frequencyBands[1]);
        features.put("high_band", frequencyBands[2]);
        return features;
    }
    private int countZeroCrossings(byte[] audioBytes) {
        int crossingCount = 0;
        boolean positive = false;
        for (int i = 0; i < audioBytes.length - 1; i += 2) {
            short sample = (short) (((audioBytes[i] & 0xFF) << 8) |
                                   (audioBytes[i + 1] & 0xFF));
            if (sample > 0 && !positive) {
                crossingCount++;
                positive = true;
            } else if (sample < 0 && positive) {
                crossingCount++;
                positive = false;
            }
        }
        return crossingCount;
    }
    private double[] analyzeFrequencyBands(byte[] audioBytes) {
        // 简化频率分析
        int samples = audioBytes.length / 2;
        double[] bands = new double[3];
        int stride = 1000; // 采样以减少计算
        int step = stride / 10;
        for (int i = 0; i < samples; i += stride) {
            // FFT在此处简化处理
            // 实际应用中应使用JTransforms或Apache Commons Math
        }
        // 简化计算：根据振幅变化估算频率
        int count = 0;
        for (int i = 0; i < audioBytes.length - 2; i += 4) {
            short sample = (short) (((audioBytes[i] & 0xFF) << 8) |
                                   (audioBytes[i + 1] & 0xFF));
            double amplitude = Math.abs(sample / 32768.0);
            if (count < bands.length) {
                bands[count % 3] += amplitude;
                count++;
            }
        }
        for (int i = 0; i < bands.length; i++) {
            if (count > 0) {
                bands[i] /= (count / 3);
            }
        }
        return bands;
    }
    private double[] vectorizeAudio(Map<String, Object> audioFeatures) {
        double[] vector = new double[10];
        vector[0] = (Double) audioFeatures.get("duration");
        vector[1] = (Double) ((Map<String, Object>) audioFeatures.get("amplitude")).get("max");
        vector[2] = (Double) audioFeatures.get("rms");
        Map<String, Object> freq = (Map<String, Object>) audioFeatures.get("frequency");
        vector[3] = ((Number) freq.get("zero_crossing_rate")).doubleValue();
        vector[4] = (Double) freq.get("low_band");
        vector[5] = (Double) freq.get("mid_band");
        vector[6] = (Double) freq.get("high_band");
        // 填充剩余维度
        for (int i = 7; i < 10; i++) {
            vector[i] = vector[i - 3];
        }
        // 归一化
        double sum = 0;
        for (double v : vector) sum += v * v;
        if (sum > 0) {
            double scale = Math.sqrt(sum);
            for (int i = 0; i < vector.length; i++) vector[i] /= scale;
        }
        return vector;
    }
    // 内部类：波形数据
    private static class AudioWaveform {
        private double[] amplitude;
        private double maxAmplitude;
        private double rms;
        public void setAmplitude(double[] amplitude) {
            this.amplitude = amplitude;
        }
        public void setMaxAmplitude(double maxAmplitude) {
            this.maxAmplitude = maxAmplitude;
        }
        public void setRMS(double rms) {
            this.rms = rms;
        }
        public Map<String, Object> getAmplitudeFeatures() {
            Map<String, Object> features = new HashMap<>();
            features.put("max", maxAmplitude);
            features.put("average", calculateAverage(amplitude));
            return features;
        }
        public double getRMS() {
            return rms;
        }
        private double calculateAverage(double[] data) {
            if (data.length == 0) return 0;
            double sum = 0;
            for (double value : data) {
                sum += Math.abs(value);
            }
            return sum / data.length;
        }
    }
}

多模态融合引擎

package com.example.multimodal.fusion;
import com.example.multimodal.core.*;
import com.example.multimodal.model.*;
import java.util.Map;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
public class MultimodalFusionEngine {
    private TextProcessor textProcessor;
    private ImageProcessor imageProcessor;
    private AudioProcessor audioProcessor;
    public MultimodalFusionEngine() {
        this.textProcessor = new TextProcessor();
        this.imageProcessor = new ImageProcessor();
        this.audioProcessor = new AudioProcessor();
    }
    public FusionResult processMultimodalData(MultimodalData data) {
        FusionResult result = new FusionResult();
        // 1. 处理各个模态
        Map<String, Object> textFeatures = null;
        Map<String, Object> imageFeatures = null;
        Map<String, Object> audioFeatures = null;
        if (data.hasText()) {
            textFeatures = textProcessor.process(data);
            System.out.println("✅ 处理文本模态完成");
        } else {
            System.out.println("⚠️ 没有文本数据");
        }
        if (data.hasImage()) {
            imageFeatures = imageProcessor.process(data);
            System.out.println("✅ 处理图像模态完成");
        } else {
            System.out.println("⚠️ 没有图像数据");
        }
        if (data.hasAudio()) {
            audioFeatures = audioProcessor.process(data);
            System.out.println("✅ 处理音频模态完成");
        } else {
            System.out.println("⚠️ 没有音频数据");
        }
        // 2. 特征融合
        fuseFeatures(result, textFeatures, imageFeatures, audioFeatures);
        // 3. 计算置信度
        Map<String, Object> confidence = new HashMap<>();
        double totalScore = 0;
        if (textFeatures != null) {
            double textScore = calculateTextScore(textFeatures);
            confidence.put("TEXT", textScore);
            totalScore += textScore;
        }
        if (imageFeatures != null) {
            double imageScore = calculateImageScore(imageFeatures);
            confidence.put("IMAGE", imageScore);
            totalScore += imageScore;
        }
        if (audioFeatures != null) {
            double audioScore = calculateAudioScore(audioFeatures);
            confidence.put("AUDIO", audioScore);
            totalScore += audioScore;
        }
        result.setConfidence(confidence);
        // 4. 确定主导模态
        result.setDominantModality(determineDominantModality(confidence));
        // 5. 综合决策（示例）
        String sentiment = determineOverallSentiment(textFeatures, imageFeatures, audioFeatures);
        result.setOverallSentiment(sentiment);
        double overallScore = (totalScore / confidence.size()) * 100;
        result.setOverallScore(overallScore);
        return result;
    }
    private void fuseFeatures(FusionResult result, 
                              Map<String, Object> textFeatures,
                              Map<String, Object> imageFeatures,
                              Map<String, Object> audioFeatures) {
        // 特征拼接和融合
        if (textFeatures != null) {
            result.addFeature("text_features", textFeatures);
        }
        if (imageFeatures != null) {
            result.addFeature("image_features", imageFeatures);
        }
        if (audioFeatures != null) {
            result.addFeature("audio_features", audioFeatures);
        }
        // 计算融合特征向量（加权平均）
        double[] fusedVector = computeFusedVector(textFeatures, imageFeatures, audioFeatures);
        result.addFeature("fused_vector", fusedVector);
    }
    private double[] computeFusedVector(Map<String, Object> textFeatures,
                                        Map<String, Object> imageFeatures,
                                        Map<String, Object> audioFeatures) {
        List<double[]> vectors = new ArrayList<>();
        if (textFeatures != null && textFeatures.containsKey("vector")) {
            vectors.add((double[]) textFeatures.get("vector"));
        }
        if (imageFeatures != null && imageFeatures.containsKey("vector")) {
            vectors.add((double[]) imageFeatures.get("vector"));
        }
        if (audioFeatures != null && audioFeatures.containsKey("vector")) {
            vectors.add((double[]) audioFeatures.get("vector"));
        }
        if (vectors.isEmpty()) {
            return new double[100];
        }
        // 拼接并取平均
        int maxLength = vectors.stream().mapToInt(v -> v.length).max().orElse(0);
        double[] fused = new double[maxLength];
        for (double[] vector : vectors) {
            if (vector.length == maxLength) {
                for (int i = 0; i < maxLength; i++) {
                    fused[i] += vector[i];
                }
            }
        }
        for (int i = 0; i < maxLength; i++) {
            fused[i] /= vectors.size();
        }
        return fused;
    }
    private double calculateTextScore(Map<String, Object> features) {
        // 简化评分逻辑
        double sentimentPositive = Math.abs((Double) features.get("sentiment_score"));
        int wordCount = (int) features.get("word_count");
        return sentimentPositive * 0.6 + Math.min(wordCount / 100.0, 1.0) * 0.4;
    }
    private double calculateImageScore(Map<String, Object> features) {
        // 简化评分逻辑
        int colors = (int) features.get("color_count");
        double brightness = (Double) features.get("brightness");
        return Math.min(colors / 500.0, 1.0) * 0.5 + 
               Math.min(brightness / 255.0, 1.0) * 0.5;
    }
    private double calculateAudioScore(Map<String, Object> features) {
        // 简化评分逻辑
        double duration = (Double) features.get("duration");
        double rms = (Double) features.get("rms");
        return Math.min(duration / 10.0, 1.0) * 0.5 + 
               Math.min(rms * 10, 1.0) * 0.5;
    }
    private String determineDominantModality(Map<String, Object> confidence) {
        String dominant = "NONE";
        double maxScore = 0;
        for (Map.Entry<String, Object> entry : confidence.entrySet()) {
            double score = (Double) entry.getValue();
            if (score > maxScore) {
                maxScore = score;
                dominant = entry.getKey();
            }
        }
        return dominant;
    }
    private String determineOverallSentiment(Map<String, Object> textFeatures,
                                             Map<String, Object> imageFeatures,
                                             Map<String, Object> audioFeatures) {
        int positiveCount = 0;
        int negativeCount = 0;
        if (textFeatures != null) {
            String sentiment = (String) textFeatures.get("sentiment_category");
            if (sentiment.equals("Positive")) positiveCount++;
            if (sentiment.equals("Negative")) negativeCount++;
        }
        if (imageFeatures != null) {
            double brightness = (Double) imageFeatures.get("brightness");
            if (brightness > 128) positiveCount++;
            else negativeCount++;
        }
        if (audioFeatures != null) {
            Map<String, Object> freq = (Map<String, Object>) audioFeatures.get("frequency");
            double highBand = (Double) freq.get("high_band");
            double lowBand = (Double) freq.get("low_band");
            if (highBand > lowBand) positiveCount++;
            else negativeCount++;
        }
        if (positiveCount > negativeCount) return "Positive";
        if (negativeCount > positiveCount) return "Negative";
        return "Neutral";
    }
}
// 辅助类
class MatrixOperations {
    public void normalizeVector(double[] vector) {
        double sum = 0;
        for (double v : vector) sum += v * v;
        if (sum > 0) {
            double scale = Math.sqrt(sum);
            for (int i = 0; i < vector.length; i++) vector[i] /= scale;
        }
    }
}

主应用程序

package com.example.multimodal;
import com.example.multimodal.fusion.MultimodalFusionEngine;
import com.example.multimodal.model.MultimodalData;
import com.example.multimodal.model.FusionResult;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.Map;
public class MultimodalApplication {
    public static void main(String[] args) {
        System.out.println("🎯 Java 多模态处理演示系统");
        System.out.println("=".repeat(50));
        // 创建多模态数据
        MultimodalData data = createSampleData();
        // 初始化融合引擎
        MultimodalFusionEngine fusionEngine = new MultimodalFusionEngine();
        try {
            // 执行多模态处理
            System.out.println("\n📊 开始多模态处理...\n");
            FusionResult result = fusionEngine.processMultimodalData(data);
            // 输出结果
            System.out.println("\n