我将为您创建一个完整的Java多模态处理案例,展示如何同时处理文本、图像和音频数据。
完整的Java多模态处理案例
项目结构
multimodal-demo/
├── pom.xml
└── src/
└── main/
└── java/
└── com/example/multimodal/
├── MultimodalApplication.java
├── core/
│ ├── ModalityProcessor.java
│ ├── TextProcessor.java
│ ├── ImageProcessor.java
│ └── AudioProcessor.java
├── fusion/
│ └── MultimodalFusionEngine.java
├── model/
│ ├── MultimodalData.java
│ └── FusionResult.java
└── util/
├── ImageUtils.java
└── AudioUtils.javaMaven依赖 (pom.xml)
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.example</groupId>
<artifactId>multimodal-demo</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<!-- OpenNLP for text processing -->
<dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-tools</artifactId>
<version>2.1.0</version>
</dependency>
<!-- OpenCV for image processing -->
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>javacv-platform</artifactId>
<version>1.5.9</version>
</dependency>
<!-- Java Sound API for audio -->
<dependency>
<groupId>com.googlecode.soundlibs</groupId>
<artifactId>tarsosdsp</artifactId>
<version>2.4</version>
</dependency>
<!-- JSON processing -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.15.2</version>
</dependency>
<!-- SLF4J for logging -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.7</version>
</dependency>
</dependencies>
</project>
核心模型类
package com.example.multimodal.model;
import java.util.HashMap;
import java.util.Map;
public class MultimodalData {
public enum ModalityType {
TEXT, IMAGE, AUDIO
}
private String text;
private byte[] imageData;
private byte[] audioData;
private Map<String, Object> metadata = new HashMap<>();
// Getters and Setters
public String getText() {
return text;
}
public void setText(String text) {
this.text = text;
if (text != null) {
metadata.put("text_length", text.length());
}
}
public byte[] getImageData() {
return imageData;
}
public void setImageData(byte[] imageData) {
this.imageData = imageData;
if (imageData != null) {
metadata.put("image_size", imageData.length);
}
}
public byte[] getAudioData() {
return audioData;
}
public void setAudioData(byte[] audioData) {
this.audioData = audioData;
if (audioData != null) {
metadata.put("audio_size", audioData.length);
}
}
public void addMetadata(String key, Object value) {
metadata.put(key, value);
}
public Object getMetadata(String key) {
return metadata.get(key);
}
public Map<String, Object> getAllMetadata() {
return metadata;
}
public boolean hasText() {
return text != null && !text.isEmpty();
}
public boolean hasImage() {
return imageData != null && imageData.length > 0;
}
public boolean hasAudio() {
return audioData != null && audioData.length > 0;
}
}
package com.example.multimodal.model;
import java.util.HashMap;
import java.util.Map;
public class FusionResult {
private Map<String, Object> features = new HashMap<>();
private Map<String, Object> confidence = new HashMap<>();
private String dominantModality;
private String overallSentiment;
private double overallScore;
public void addFeature(String key, Object value) {
features.put(key, value);
}
public void addConfidence(String modality, Double value) {
confidence.put(modality, value);
}
// Getters and Setters
public Map<String, Object> getFeatures() {
return features;
}
public void setFeatures(Map<String, Object> features) {
this.features = features;
}
public Map<String, Object> getConfidence() {
return confidence;
}
public void setConfidence(Map<String, Object> confidence) {
this.confidence = confidence;
}
public String getDominantModality() {
return dominantModality;
}
public void setDominantModality(String dominantModality) {
this.dominantModality = dominantModality;
}
public String getOverallSentiment() {
return overallSentiment;
}
public void setOverallSentiment(String overallSentiment) {
this.overallSentiment = overallSentiment;
}
public double getOverallScore() {
return overallScore;
}
public void setOverallScore(double overallScore) {
this.overallScore = overallScore;
}
}
核心处理器
package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
public interface ModalityProcessor {
String getModalityType();
Object process(MultimodalData data);
}
package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
public class TextProcessor implements ModalityProcessor {
private MatrixOperations matrixOps;
public TextProcessor() {
this.matrixOps = new MatrixOperations();
}
@Override
public String getModalityType() {
return "TEXT";
}
@Override
public Map<String, Object> process(MultimodalData data) {
if (!data.hasText()) {
return null;
}
String text = data.getText();
Map<String, Object> textFeatures = new HashMap<>();
// 1. 文本统计特征
textFeatures.put("word_count", countWords(text));
textFeatures.put("char_count", text.length());
textFeatures.put("sentence_count", countSentences(text));
textFeatures.put("unique_words", getUniqueWords(text));
// 2. 情感分析（简化版）
textFeatures.put("sentiment_score", analyzeSentiment(text));
textFeatures.put("sentiment_category", getSentimentCategory(
(double) textFeatures.get("sentiment_score")));
// 3. 关键词提取（简化版）
textFeatures.put("keywords", extractKeywords(text));
// 4. 文本向量化（模拟Word2Vec）
double[] textVector = vectorizeText(text);
textFeatures.put("vector", textVector);
data.addMetadata("text_features", textFeatures);
data.addMetadata("text_vector", textVector);
return textFeatures;
}
private int countWords(String text) {
return text.trim().split("\\s+").length;
}
private int countSentences(String text) {
return text.split("[.!?]").length;
}
private int getUniqueWords(String text) {
return (int) Arrays.stream(text.toLowerCase().split("\\s+"))
.distinct()
.count();
}
private double analyzeSentiment(String text) {
String[] positiveWords = {"good", "great", "excellent", "wonderful", "amazing",
"happy", "love", "like", "beautiful", "awesome"};
String[] negativeWords = {"bad", "terrible", "awful", "horrible", "hate",
"ugly", "poor", "worst", "worried", "sad"};
String[] words = text.toLowerCase().split("\\s+");
int positiveCount = 0;
int negativeCount = 0;
for (String word : words) {
if (Arrays.asList(positiveWords).contains(word)) {
positiveCount++;
}
if (Arrays.asList(negativeWords).contains(word)) {
negativeCount++;
}
}
return (positiveCount - negativeCount) / (double) words.length;
}
private String getSentimentCategory(double score) {
if (score > 0.1) return "Positive";
if (score < -0.1) return "Negative";
return "Neutral";
}
private String[] extractKeywords(String text) {
// 简化提取：过滤停用词后取高频词
String[] stopWords = {"the", "a", "an", "and", "or", "but", "is", "are",
"was", "were", "be", "been", "being"};
return Arrays.stream(text.toLowerCase().split("\\s+"))
.filter(word -> !Arrays.asList(stopWords).contains(word))
.distinct()
.limit(10)
.toArray(String[]::new);
}
private double[] vectorizeText(String text) {
// 模拟文本向量化，实际应用中会使用预训练模型
String[] words = text.toLowerCase().split("\\s+");
double[] vector = new double[100];
for (String word : words) {
// 使用哈希函数模拟词嵌入
int hashCode = Math.abs(word.hashCode());
int index = hashCode % vector.length;
vector[index] += 1.0;
}
// 归一化
matrixOps.normalizeVector(vector);
return vector;
}
}
图像处理器
package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
import com.example.multimodal.util.ImageUtils;
import org.bytedeco.javacv.Java2DFrameConverter;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.bytedeco.opencv.opencv_core.Mat;
import org.bytedeco.opencv.opencv_core.Scalar;
import org.bytedeco.opencv.opencv_imgproc.Vec2d;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Map;
public class ImageProcessor implements ModalityProcessor {
private ImageUtils imageUtils;
public ImageProcessor() {
this.imageUtils = new ImageUtils();
}
@Override
public String getModalityType() {
return "IMAGE";
}
@Override
public Map<String, Object> process(MultimodalData data) {
if (!data.hasImage()) {
return null;
}
try {
byte[] imageBytes = data.getImageData();
BufferedImage image = ImageIO.read(new ByteArrayInputStream(imageBytes));
if (image == null) {
return null;
}
Map<String, Object> imageFeatures = new HashMap<>();
// 1. 基本图像特征
imageFeatures.put("width", image.getWidth());
imageFeatures.put("height", image.getHeight());
imageFeatures.put("color_count", countColors(image));
imageFeatures.put("brightness", calculateAverageBrightness(image));
// 2. 颜色特征
double[] colorHistogram = extractColorHistogram(image);
imageFeatures.put("color_histogram", colorHistogram);
// 3. 纹理特征（简化版）
Map<String, Object> textureFeatures = extractTextureFeatures(image);
imageFeatures.put("texture", textureFeatures);
// 4. 图像向量化
double[] imageVector = vectorizeImage(image);
imageFeatures.put("vector", imageVector);
data.addMetadata("image_features", imageFeatures);
data.addMetadata("image_vector", imageVector);
return imageFeatures;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
private int countColors(BufferedImage image) {
java.util.Set<Integer> colors = new java.util.HashSet<>();
int stride = 5; // 采样以提高性能
for (int x = 0; x < image.getWidth(); x += stride) {
for (int y = 0; y < image.getHeight(); y += stride) {
colors.add(image.getRGB(x, y));
}
}
return colors.size();
}
private double calculateAverageBrightness(BufferedImage image) {
long sum = 0;
int count = 0;
int stride = 5;
for (int x = 0; x < image.getWidth(); x += stride) {
for (int y = 0; y < image.getHeight(); y += stride) {
java.awt.Color c = new java.awt.Color(image.getRGB(x, y));
sum += (c.getRed() + c.getGreen() + c.getBlue()) / 3;
count++;
}
}
return sum / (double) count;
}
private double[] extractColorHistogram(BufferedImage image) {
double[] histogram = new double[64]; // 使用量化颜色空间
int bucketsPerChannel = 4; // 4^3 = 64 buckets
int stride = 3;
for (int x = 0; x < image.getWidth(); x += stride) {
for (int y = 0; y < image.getHeight(); y += stride) {
java.awt.Color c = new java.awt.Color(image.getRGB(x, y));
int r = c.getRed() / 64; // 4 buckets for red
int g = c.getGreen() / 64;
int b = c.getBlue() / 64;
int index = (r * bucketsPerChannel * bucketsPerChannel) +
(g * bucketsPerChannel) + b;
histogram[index]++;
}
}
// 归一化直方图
double sum = 0;
for (double value : histogram) sum += value;
if (sum > 0) {
for (int i = 0; i < histogram.length; i++) {
histogram[i] /= sum;
}
}
return histogram;
}
private Map<String, Object> extractTextureFeatures(BufferedImage image) {
Map<String, Object> features = new HashMap<>();
// 计算灰度共生矩阵（简化的统计特征）
double[] grayLevels = new double[256];
int count = 0;
int stride = 5;
for (int x = 0; x < image.getWidth(); x += stride) {
for (int y = 0; y < image.getHeight(); y += stride) {
java.awt.Color c = new java.awt.Color(image.getRGB(x, y));
int gray = (int) (0.299 * c.getRed() +
0.587 * c.getGreen() +
0.114 * c.getBlue());
grayLevels[gray]++;
count++;
}
}
// 计算统计特征
double mean = 0;
double stdDev = 0;
for (int i = 0; i < 256; i++) {
double probability = grayLevels[i] / count;
mean += i * probability;
}
for (int i = 0; i < 256; i++) {
double probability = grayLevels[i] / count;
stdDev += (i - mean) * (i - mean) * probability;
}
stdDev = Math.sqrt(stdDev);
features.put("mean_gray", mean);
features.put("std_dev", stdDev);
features.put("smoothness", 1.0 / (1.0 + stdDev * stdDev));
// 能量和熵
double energy = 0;
double entropy = 0;
for (int i = 0; i < 256; i++) {
double probability = grayLevels[i] / count;
if (probability > 0) {
energy += probability * probability;
entropy -= probability * (Math.log(probability) / Math.log(2));
}
}
features.put("energy", energy);
features.put("entropy", entropy);
return features;
}
private double[] vectorizeImage(BufferedImage image) {
// 转换图像为特征向量（使用颜色直方图和空间特征）
double[] colorHist = extractColorHistogram(image);
Map<String, Object> texture = extractTextureFeatures(image);
// 组合特征
double[] vector = new double[70];
System.arraycopy(colorHist, 0, vector, 0, colorHist.length);
vector[64] = (Double) texture.get("mean_gray");
vector[65] = (Double) texture.get("std_dev");
vector[66] = (Double) texture.get("smoothness");
vector[67] = (Double) texture.get("energy");
vector[68] = (Double) texture.get("entropy");
vector[69] = (Double) texture.get("entropy");
// 归一化
double sum = 0;
for (double v : vector) sum += v * v;
if (sum > 0) {
double scale = Math.sqrt(sum);
for (int i = 0; i < vector.length; i++) vector[i] /= scale;
}
return vector;
}
}
音频处理器
package com.example.multimodal.core;
import com.example.multimodal.model.MultimodalData;
import javax.sound.sampled.*;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Map;
public class AudioProcessor implements ModalityProcessor {
@Override
public String getModalityType() {
return "AUDIO";
}
@Override
public Map<String, Object> process(MultimodalData data) {
if (!data.hasAudio()) {
return null;
}
try {
byte[] audioBytes = data.getAudioData();
Map<String, Object> audioFeatures = new HashMap<>();
// 1. 基本信息
audioFeatures.put("duration", calculateDuration(audioBytes));
audioFeatures.put("sample_rate", 44100); // 典型采样率
audioFeatures.put("channels", 2);
// 2. 波形特征
AudioWaveform waveform = extractWaveform(audioBytes);
audioFeatures.put("amplitude", waveform.getAmplitudeFeatures());
audioFeatures.put("rms", waveform.getRMS());
// 3. 频域特征
Map<String, Object> frequencyFeatures = extractFrequencyFeatures(audioBytes);
audioFeatures.put("frequency", frequencyFeatures);
// 4. 音频向量化
double[] audioVector = vectorizeAudio(audioFeatures);
audioFeatures.put("vector", audioVector);
data.addMetadata("audio_features", audioFeatures);
data.addMetadata("audio_vector", audioVector);
return audioFeatures;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
private double calculateDuration(byte[] audioBytes) {
// 假设：16位PCM，44100Hz，双声道 -> Bytes = samples * 2 * 2
int sampleSize = 2; // 16位
int channels = 2;
int sampleRate = 44100;
long totalSamples = audioBytes.length / (sampleSize * channels);
return totalSamples / (double) sampleRate;
}
private AudioWaveform extractWaveform(byte[] audioBytes) {
AudioWaveform waveform = new AudioWaveform();
int samples = audioBytes.length / 2;
double[] amplitudes = new double[samples / 100]; // 降采样
double maxAmplitude = 0;
double sumSquares = 0;
int count = 0;
for (int i = 0; i < samples; i += 100) {
int startSample = i * 2;
if (startSample + 1 < audioBytes.length) {
short sample = (short) (((audioBytes[startSample] & 0xFF) << 8) |
(audioBytes[startSample + 1] & 0xFF));
double amplitude = sample / 32768.0;
if (count < amplitudes.length) {
amplitudes[count] = amplitude;
count++;
}
if (Math.abs(amplitude) > maxAmplitude) {
maxAmplitude = Math.abs(amplitude);
}
sumSquares += amplitude * amplitude;
}
}
waveform.setAmplitude(amplitudes);
waveform.setMaxAmplitude(maxAmplitude);
waveform.setRMS(Math.sqrt(sumSquares / samples));
return waveform;
}
private Map<String, Object> extractFrequencyFeatures(byte[] audioBytes) {
Map<String, Object> features = new HashMap<>();
// 简化的FFT分析（计算过零率等）
int zeroCrossings = countZeroCrossings(audioBytes);
features.put("zero_crossing_rate", zeroCrossings);
double[] frequencyBands = analyzeFrequencyBands(audioBytes);
features.put("low_band", frequencyBands[0]);
features.put("mid_band", frequencyBands[1]);
features.put("high_band", frequencyBands[2]);
return features;
}
private int countZeroCrossings(byte[] audioBytes) {
int crossingCount = 0;
boolean positive = false;
for (int i = 0; i < audioBytes.length - 1; i += 2) {
short sample = (short) (((audioBytes[i] & 0xFF) << 8) |
(audioBytes[i + 1] & 0xFF));
if (sample > 0 && !positive) {
crossingCount++;
positive = true;
} else if (sample < 0 && positive) {
crossingCount++;
positive = false;
}
}
return crossingCount;
}
private double[] analyzeFrequencyBands(byte[] audioBytes) {
// 简化频率分析
int samples = audioBytes.length / 2;
double[] bands = new double[3];
int stride = 1000; // 采样以减少计算
int step = stride / 10;
for (int i = 0; i < samples; i += stride) {
// FFT在此处简化处理
// 实际应用中应使用JTransforms或Apache Commons Math
}
// 简化计算：根据振幅变化估算频率
int count = 0;
for (int i = 0; i < audioBytes.length - 2; i += 4) {
short sample = (short) (((audioBytes[i] & 0xFF) << 8) |
(audioBytes[i + 1] & 0xFF));
double amplitude = Math.abs(sample / 32768.0);
if (count < bands.length) {
bands[count % 3] += amplitude;
count++;
}
}
for (int i = 0; i < bands.length; i++) {
if (count > 0) {
bands[i] /= (count / 3);
}
}
return bands;
}
private double[] vectorizeAudio(Map<String, Object> audioFeatures) {
double[] vector = new double[10];
vector[0] = (Double) audioFeatures.get("duration");
vector[1] = (Double) ((Map<String, Object>) audioFeatures.get("amplitude")).get("max");
vector[2] = (Double) audioFeatures.get("rms");
Map<String, Object> freq = (Map<String, Object>) audioFeatures.get("frequency");
vector[3] = ((Number) freq.get("zero_crossing_rate")).doubleValue();
vector[4] = (Double) freq.get("low_band");
vector[5] = (Double) freq.get("mid_band");
vector[6] = (Double) freq.get("high_band");
// 填充剩余维度
for (int i = 7; i < 10; i++) {
vector[i] = vector[i - 3];
}
// 归一化
double sum = 0;
for (double v : vector) sum += v * v;
if (sum > 0) {
double scale = Math.sqrt(sum);
for (int i = 0; i < vector.length; i++) vector[i] /= scale;
}
return vector;
}
// 内部类：波形数据
private static class AudioWaveform {
private double[] amplitude;
private double maxAmplitude;
private double rms;
public void setAmplitude(double[] amplitude) {
this.amplitude = amplitude;
}
public void setMaxAmplitude(double maxAmplitude) {
this.maxAmplitude = maxAmplitude;
}
public void setRMS(double rms) {
this.rms = rms;
}
public Map<String, Object> getAmplitudeFeatures() {
Map<String, Object> features = new HashMap<>();
features.put("max", maxAmplitude);
features.put("average", calculateAverage(amplitude));
return features;
}
public double getRMS() {
return rms;
}
private double calculateAverage(double[] data) {
if (data.length == 0) return 0;
double sum = 0;
for (double value : data) {
sum += Math.abs(value);
}
return sum / data.length;
}
}
}
多模态融合引擎
package com.example.multimodal.fusion;
import com.example.multimodal.core.*;
import com.example.multimodal.model.*;
import java.util.Map;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
public class MultimodalFusionEngine {
private TextProcessor textProcessor;
private ImageProcessor imageProcessor;
private AudioProcessor audioProcessor;
public MultimodalFusionEngine() {
this.textProcessor = new TextProcessor();
this.imageProcessor = new ImageProcessor();
this.audioProcessor = new AudioProcessor();
}
public FusionResult processMultimodalData(MultimodalData data) {
FusionResult result = new FusionResult();
// 1. 处理各个模态
Map<String, Object> textFeatures = null;
Map<String, Object> imageFeatures = null;
Map<String, Object> audioFeatures = null;
if (data.hasText()) {
textFeatures = textProcessor.process(data);
System.out.println("✅ 处理文本模态完成");
} else {
System.out.println("⚠️ 没有文本数据");
}
if (data.hasImage()) {
imageFeatures = imageProcessor.process(data);
System.out.println("✅ 处理图像模态完成");
} else {
System.out.println("⚠️ 没有图像数据");
}
if (data.hasAudio()) {
audioFeatures = audioProcessor.process(data);
System.out.println("✅ 处理音频模态完成");
} else {
System.out.println("⚠️ 没有音频数据");
}
// 2. 特征融合
fuseFeatures(result, textFeatures, imageFeatures, audioFeatures);
// 3. 计算置信度
Map<String, Object> confidence = new HashMap<>();
double totalScore = 0;
if (textFeatures != null) {
double textScore = calculateTextScore(textFeatures);
confidence.put("TEXT", textScore);
totalScore += textScore;
}
if (imageFeatures != null) {
double imageScore = calculateImageScore(imageFeatures);
confidence.put("IMAGE", imageScore);
totalScore += imageScore;
}
if (audioFeatures != null) {
double audioScore = calculateAudioScore(audioFeatures);
confidence.put("AUDIO", audioScore);
totalScore += audioScore;
}
result.setConfidence(confidence);
// 4. 确定主导模态
result.setDominantModality(determineDominantModality(confidence));
// 5. 综合决策（示例）
String sentiment = determineOverallSentiment(textFeatures, imageFeatures, audioFeatures);
result.setOverallSentiment(sentiment);
double overallScore = (totalScore / confidence.size()) * 100;
result.setOverallScore(overallScore);
return result;
}
private void fuseFeatures(FusionResult result,
Map<String, Object> textFeatures,
Map<String, Object> imageFeatures,
Map<String, Object> audioFeatures) {
// 特征拼接和融合
if (textFeatures != null) {
result.addFeature("text_features", textFeatures);
}
if (imageFeatures != null) {
result.addFeature("image_features", imageFeatures);
}
if (audioFeatures != null) {
result.addFeature("audio_features", audioFeatures);
}
// 计算融合特征向量（加权平均）
double[] fusedVector = computeFusedVector(textFeatures, imageFeatures, audioFeatures);
result.addFeature("fused_vector", fusedVector);
}
private double[] computeFusedVector(Map<String, Object> textFeatures,
Map<String, Object> imageFeatures,
Map<String, Object> audioFeatures) {
List<double[]> vectors = new ArrayList<>();
if (textFeatures != null && textFeatures.containsKey("vector")) {
vectors.add((double[]) textFeatures.get("vector"));
}
if (imageFeatures != null && imageFeatures.containsKey("vector")) {
vectors.add((double[]) imageFeatures.get("vector"));
}
if (audioFeatures != null && audioFeatures.containsKey("vector")) {
vectors.add((double[]) audioFeatures.get("vector"));
}
if (vectors.isEmpty()) {
return new double[100];
}
// 拼接并取平均
int maxLength = vectors.stream().mapToInt(v -> v.length).max().orElse(0);
double[] fused = new double[maxLength];
for (double[] vector : vectors) {
if (vector.length == maxLength) {
for (int i = 0; i < maxLength; i++) {
fused[i] += vector[i];
}
}
}
for (int i = 0; i < maxLength; i++) {
fused[i] /= vectors.size();
}
return fused;
}
private double calculateTextScore(Map<String, Object> features) {
// 简化评分逻辑
double sentimentPositive = Math.abs((Double) features.get("sentiment_score"));
int wordCount = (int) features.get("word_count");
return sentimentPositive * 0.6 + Math.min(wordCount / 100.0, 1.0) * 0.4;
}
private double calculateImageScore(Map<String, Object> features) {
// 简化评分逻辑
int colors = (int) features.get("color_count");
double brightness = (Double) features.get("brightness");
return Math.min(colors / 500.0, 1.0) * 0.5 +
Math.min(brightness / 255.0, 1.0) * 0.5;
}
private double calculateAudioScore(Map<String, Object> features) {
// 简化评分逻辑
double duration = (Double) features.get("duration");
double rms = (Double) features.get("rms");
return Math.min(duration / 10.0, 1.0) * 0.5 +
Math.min(rms * 10, 1.0) * 0.5;
}
private String determineDominantModality(Map<String, Object> confidence) {
String dominant = "NONE";
double maxScore = 0;
for (Map.Entry<String, Object> entry : confidence.entrySet()) {
double score = (Double) entry.getValue();
if (score > maxScore) {
maxScore = score;
dominant = entry.getKey();
}
}
return dominant;
}
private String determineOverallSentiment(Map<String, Object> textFeatures,
Map<String, Object> imageFeatures,
Map<String, Object> audioFeatures) {
int positiveCount = 0;
int negativeCount = 0;
if (textFeatures != null) {
String sentiment = (String) textFeatures.get("sentiment_category");
if (sentiment.equals("Positive")) positiveCount++;
if (sentiment.equals("Negative")) negativeCount++;
}
if (imageFeatures != null) {
double brightness = (Double) imageFeatures.get("brightness");
if (brightness > 128) positiveCount++;
else negativeCount++;
}
if (audioFeatures != null) {
Map<String, Object> freq = (Map<String, Object>) audioFeatures.get("frequency");
double highBand = (Double) freq.get("high_band");
double lowBand = (Double) freq.get("low_band");
if (highBand > lowBand) positiveCount++;
else negativeCount++;
}
if (positiveCount > negativeCount) return "Positive";
if (negativeCount > positiveCount) return "Negative";
return "Neutral";
}
}
// 辅助类
class MatrixOperations {
public void normalizeVector(double[] vector) {
double sum = 0;
for (double v : vector) sum += v * v;
if (sum > 0) {
double scale = Math.sqrt(sum);
for (int i = 0; i < vector.length; i++) vector[i] /= scale;
}
}
}
主应用程序
package com.example.multimodal;
import com.example.multimodal.fusion.MultimodalFusionEngine;
import com.example.multimodal.model.MultimodalData;
import com.example.multimodal.model.FusionResult;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.Map;
public class MultimodalApplication {
public static void main(String[] args) {
System.out.println("🎯 Java 多模态处理演示系统");
System.out.println("=".repeat(50));
// 创建多模态数据
MultimodalData data = createSampleData();
// 初始化融合引擎
MultimodalFusionEngine fusionEngine = new MultimodalFusionEngine();
try {
// 执行多模态处理
System.out.println("\n📊 开始多模态处理...\n");
FusionResult result = fusionEngine.processMultimodalData(data);
// 输出结果
System.out.println("\n
