脚本如何实现文件内容支持向量机

本文目录导读：

1. Python实现（最常用）
2. Shell脚本辅助实现
3. 完整的文件处理脚本
4. 使用示例
5. 关键要点

的支持向量机（SVM）分类。

Python实现（最常用）

基础文本SVM分类器

import numpy as np
from sklearn import svm
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import os
class FileContentSVM:
    def __init__(self, kernel='linear'):
        self.vectorizer = TfidfVectorizer(max_features=5000)
        self.svm_classifier = svm.SVC(kernel=kernel, C=1.0)
    def load_files_from_directory(self, directory_path):
        """从目录加载文件内容"""
        texts = []
        labels = []
        for filename in os.listdir(directory_path):
            file_path = os.path.join(directory_path, filename)
            if os.path.isfile(file_path):
                try:
                    with open(file_path, 'r', encoding='utf-8') as f:
                        content = f.read()
                        texts.append(content)
                        # 根据目录名或文件名确定标签
                        labels.append(directory_path.split('/')[-1])
                except Exception as e:
                    print(f"Error reading {file_path}: {e}")
        return texts, labels
    def train(self, texts, labels):
        """训练SVM模型"""
        # 转换文本为特征向量
        X = self.vectorizer.fit_transform(texts)
        y = np.array(labels)
        # 划分训练集和测试集
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.2, random_state=42
        )
        # 训练SVM模型
        self.svm_classifier.fit(X_train, y_train)
        # 评估模型
        y_pred = self.svm_classifier.predict(X_test)
        print("Classification Report:")
        print(classification_report(y_test, y_pred))
        return X_train, X_test, y_train, y_test
    def predict(self, text):
        """预测新文本的类别"""
        X = self.vectorizer.transform([text])
        return self.svm_classifier.predict(X)[0]
    def save_model(self, model_path):
        """保存模型"""
        import joblib
        joblib.dump({
            'vectorizer': self.vectorizer,
            'classifier': self.svm_classifier
        }, model_path)
    def load_model(self, model_path):
        """加载模型"""
        import joblib
        model_data = joblib.load(model_path)
        self.vectorizer = model_data['vectorizer']
        self.svm_classifier = model_data['classifier']
# 使用示例
def text_classification_example():
    # 准备示例数据
    texts = [
        "机器学习是人工智能的重要分支",
        "深度学习需要大量计算资源",
        "今天的天气很好适合出游",
        "股票市场今天大涨",
    ]
    labels = ["科技", "科技", "生活", "财经"]
    # 初始化并训练模型
    svm_model = FileContentSVM(kernel='linear')
    svm_model.train(texts, labels)
    # 预测新文本
    new_text = "人工智能技术发展迅速"
    prediction = svm_model.predict(new_text)
    print(f"预测结果: {prediction}")
# text_classification_example()

Shell脚本辅助实现

#!/bin/bash
SVM分类器的Shell辅助脚本
# 需要Python环境支持
# 配置
TRAIN_DIR="./training_data"
TEST_DIR="./test_data"
MODEL_PATH="./svm_model.pkl"
# 检查Python是否安装
check_python() {
    if ! command -v python3 &> /dev/null; then
        echo "Error: Python3 is required"
        exit 1
    fi
    # 检查必要的Python包
    python3 -c "import sklearn" 2>/dev/null || {
        echo "Installing required packages..."
        pip3 install scikit-learn numpy joblib
    }
}
# 准备训练数据
prepare_training_data() {
    local category=$1
    local source_dir=$2
    mkdir -p "$TRAIN_DIR/$category"
    cp "$source_dir"/*.txt "$TRAIN_DIR/$category/" 2>/dev/null
    echo "Training data prepared for category: $category"
}
# 训练模型
train_model() {
    python3 << EOF
import sys
sys.path.insert(0, '.')
from svm_classifier import FileContentSVM
import os
# 加载训练数据
svm_model = FileContentSVM()
all_texts = []
all_labels = []
for category in os.listdir('$TRAIN_DIR'):
    category_path = os.path.join('$TRAIN_DIR', category)
    if os.path.isdir(category_path):
        texts, labels = svm_model.load_files_from_directory(category_path)
        all_texts.extend(texts)
        all_labels.extend(labels)
# 训练
svm_model.train(all_texts, all_labels)
svm_model.save_model('$MODEL_PATH')
print("Model trained and saved successfully")
EOF
}
# 预测单个文件
predict_file() {
    local file_path=$1
    python3 << EOF
import sys
sys.path.insert(0, '.')
from svm_classifier import FileContentSVM
svm_model = FileContentSVM()
svm_model.load_model('$MODEL_PATH')
with open('$file_path', 'r') as f:
    content = f.read()
prediction = svm_model.predict(content)
print(f"File: $file_path")
print(f"Prediction: {prediction}")
EOF
}
# 批量预测
batch_predict() {
    local directory=$1
    for file in "$directory"/*; do
        if [ -f "$file" ]; then
            predict_file "$file"
            echo "---"
        fi
    done
}
# 主函数
main() {
    check_python
    case $1 in
        prepare)
            prepare_training_data "$2" "$3"
            ;;
        train)
            train_model
            ;;
        predict)
            predict_file "$2"
            ;;
        batch)
            batch_predict "$2"
            ;;
        *)
            echo "Usage: $0 {prepare|train|predict|batch} [args]"
            echo "  prepare <category> <source_dir>"
            echo "  train"
            echo "  predict <file_path>"
            echo "  batch <directory>"
            ;;
    esac
}
# 执行主函数
main "$@"

完整的文件处理脚本

#!/usr/bin/env python3
import os
import sys
import argparse
import logging
from pathlib import Path
from typing import List, Tuple
import numpy as np
from sklearn import svm
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
import joblib
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class FileContentSVMClassifier:
    """完整的文件内容SVM分类器"""
    def __init__(self, config=None):
        self.config = config or {
            'max_features': 5000,
            'kernel': 'linear',
            'C': 1.0,
            'test_size': 0.2,
            'random_state': 42
        }
        self.vectorizer = TfidfVectorizer(
            max_features=self.config['max_features'],
            ngram_range=(1, 2),
            analyzer='word',
            stop_words=None
        )
        self.classifier = svm.SVC(
            kernel=self.config['kernel'],
            C=self.config['C'],
            probability=True
        )
        self.label_encoder = LabelEncoder()
    def load_training_data(self, data_path: str) -> Tuple[List[str], List[str]]:
        """加载训练数据"""
        texts = []
        labels = []
        data_path = Path(data_path)
        # 支持多种数据格式
        if data_path.is_dir():
            # 从目录加载（每个子目录代表一个类别）
            for category_dir in data_path.iterdir():
                if category_dir.is_dir():
                    category = category_dir.name
                    for file_path in category_dir.glob("*"):
                        if file_path.is_file():
                            try:
                                with open(file_path, 'r', encoding='utf-8') as f:
                                    content = f.read()
                                    texts.append(content)
                                    labels.append(category)
                            except Exception as e:
                                logger.error(f"Error reading {file_path}: {e}")
        elif data_path.is_file() and data_path.suffix == '.csv':
            # 从CSV文件加载
            import pandas as pd
            df = pd.read_csv(data_path)
            texts = df['text'].tolist()
            labels = df['label'].tolist()
        return texts, labels
    def train(self, texts: List[str], labels: List[str]):
        """训练模型"""
        logger.info("Starting training...")
        # 编码标签
        y = self.label_encoder.fit_transform(labels)
        # 文本向量化
        X = self.vectorizer.fit_transform(texts)
        # 划分训练集和测试集
        from sklearn.model_selection import train_test_split
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, 
            test_size=self.config['test_size'],
            random_state=self.config['random_state'],
            stratify=y
        )
        # 训练
        self.classifier.fit(X_train, y_train)
        # 评估
        accuracy = self.classifier.score(X_test, y_test)
        logger.info(f"Training completed. Accuracy: {accuracy:.4f}")
        return X_train, X_test, y_train, y_test
    def predict(self, text: str) -> Tuple[str, float]:
        """预测文本类别"""
        X = self.vectorizer.transform([text])
        # 获取预测结果和概率
        prediction = self.classifier.predict(X)[0]
        probabilities = self.classifier.predict_proba(X)[0]
        confidence = max(probabilities)
        # 解码标签
        predicted_label = self.label_encoder.inverse_transform([prediction])[0]
        return predicted_label, confidence
    def batch_predict(self, directory: str) -> List[Tuple[str, str, float]]:
        """批量预测文件"""
        results = []
        directory = Path(directory)
        for file_path in directory.glob("*"):
            if file_path.is_file():
                try:
                    with open(file_path, 'r', encoding='utf-8') as f:
                        content = f.read()
                    label, confidence = self.predict(content)
                    results.append((file_path.name, label, confidence))
                    logger.info(f"Predicted {file_path.name}: {label} ({confidence:.2f})")
                except Exception as e:
                    logger.error(f"Error processing {file_path}: {e}")
        return results
    def save(self, save_path: str):
        """保存模型"""
        model_data = {
            'vectorizer': self.vectorizer,
            'classifier': self.classifier,
            'label_encoder': self.label_encoder,
            'config': self.config
        }
        joblib.dump(model_data, save_path)
        logger.info(f"Model saved to {save_path}")
    def load(self, load_path: str):
        """加载模型"""
        model_data = joblib.load(load_path)
        self.vectorizer = model_data['vectorizer']
        self.classifier = model_data['classifier']
        self.label_encoder = model_data['label_encoder']
        self.config = model_data.get('config', self.config)
        logger.info(f"Model loaded from {load_path}")
def main():
    parser = argparse.ArgumentParser(description='文件内容SVM分类器')
    parser.add_argument('mode', choices=['train', 'predict', 'batch'], 
                       help='运行模式')
    parser.add_argument('--data', '-d', help='训练数据路径')
    parser.add_argument('--model', '-m', default='svm_model.pkl', 
                       help='模型文件路径')
    parser.add_argument('--input', '-i', help='输入文件或目录')
    parser.add_argument('--output', '-o', help='输出文件')
    args = parser.parse_args()
    classifier = FileContentSVMClassifier()
    if args.mode == 'train':
        if not args.data:
            parser.error("训练模式需要指定--data参数")
        # 加载训练数据
        texts, labels = classifier.load_training_data(args.data)
        if not texts:
            logger.error("No training data found")
            sys.exit(1)
        logger.info(f"Loaded {len(texts)} training samples")
        # 训练
        classifier.train(texts, labels)
        # 保存模型
        classifier.save(args.model)
    elif args.mode == 'predict':
        if not args.input:
            parser.error("预测模式需要指定--input参数")
        # 加载模型
        classifier.load(args.model)
        # 预测
        with open(args.input, 'r') as f:
            content = f.read()
        label, confidence = classifier.predict(content)
        print(f"预测结果: {label}")
        print(f"置信度: {confidence:.2f}")
        if args.output:
            with open(args.output, 'w') as f:
                f.write(f"Label: {label}\n")
                f.write(f"Confidence: {confidence:.2f}\n")
    elif args.mode == 'batch':
        if not args.input:
            parser.error("批量预测需要指定--input参数")
        # 加载模型
        classifier.load(args.model)
        # 批量预测
        results = classifier.batch_predict(args.input)
        # 输出结果
        if args.output:
            with open(args.output, 'w') as f:
                f.write("文件名,预测类别,置信度\n")
                for filename, label, confidence in results:
                    f.write(f"{filename},{label},{confidence:.2f}\n")
        else:
            for filename, label, confidence in results:
                print(f"{filename}: {label} ({confidence:.2f})")
if __name__ == "__main__":
    main()

使用示例

# 1. 训练模型
python3 svm_classifier.py train --data ./training_data --model my_model.pkl
# 2. 预测单个文件
python3 svm_classifier.py predict --input ./test.txt --model my_model.pkl
# 3. 批量预测
python3 svm_classifier.py batch --input ./test_directory --model my_model.pkl --output results.csv
# 4. 使用Shell辅助脚本
chmod +x svm_helper.sh
./svm_helper.sh prepare tech ./tech_files
./svm_helper.sh train
./svm_helper.sh predict ./new_document.txt

关键要点

1. 文本预处理：使用TF-IDF向量化提取特征
2. 模型选择：线性核适合文本分类，RBF核适合非线性问题
3. 参数调优：调整C值、核函数等参数提高准确率
4. 性能优化：限制特征数量，使用稀疏矩阵
5. 持久化：保存训练好的模型供后续使用

这个实现可以根据你的具体需求进行调整和扩展。