Java故障演练案例

本文目录导读：

1. 故障演练框架设计
2. 常见故障场景实现
3. 业务系统模拟
4. 监控和告警系统
5. 故障演练执行器
6. 故障演练结果分析
7. 使用说明
8. 安全建议

我将为您提供一个完整的Java故障演练案例,涵盖常见的故障场景和演练方法。

故障演练框架设计

package com.example.faultsimulator;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class FaultSimulator {
    private static volatile FaultSimulator instance;
    private final Map<String, FaultRule> faultRules = new ConcurrentHashMap<>();
    private final AtomicBoolean enabled = new AtomicBoolean(false);
    private FaultSimulator() {}
    public static FaultSimulator getInstance() {
        if (instance == null) {
            synchronized (FaultSimulator.class) {
                if (instance == null) {
                    instance = new FaultSimulator();
                }
            }
        }
        return instance;
    }
    /**
     * 注册故障规则
     */
    public void registerFaultRule(String name, FaultRule rule) {
        faultRules.put(name, rule);
        enabled.set(true);
    }
    /**
     * 模拟故障触发
     */
    public void triggerFault(String ruleName) {
        FaultRule rule = faultRules.get(ruleName);
        if (rule != null && enabled.get()) {
            rule.execute();
        }
    }
    public void disableAllFaults() {
        enabled.set(false);
        faultRules.clear();
    }
    /**
     * 故障规则接口
     */
    public interface FaultRule {
        void execute();
    }
}

常见故障场景实现

package com.example.faultsimulator.scenarios;
import com.example.faultsimulator.FaultSimulator;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.Random;
public class CommonFaultScenarios {
    /**
     * 场景1: CPU飙升故障
     */
    public static class CPUHighUsageFault implements FaultSimulator.FaultRule {
        private volatile boolean running = true;
        private ExecutorService executor;
        @Override
        public void execute() {
            int cores = Runtime.getRuntime().availableProcessors();
            executor = Executors.newFixedThreadPool(cores);
            for (int i = 0; i < cores; i++) {
                executor.submit(() -> {
                    while (running) {
                        // 死循环消耗CPU
                        Math.pow(Math.random(), Math.random());
                    }
                });
            }
            // 自动恢复定时器
            ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
            scheduler.schedule(() -> stop(), 30, TimeUnit.SECONDS);
        }
        public void stop() {
            running = false;
            if (executor != null) {
                executor.shutdownNow();
            }
        }
    }
    /**
     * 场景2: 内存溢出故障
     */
    public static class MemoryLeakFault implements FaultSimulator.FaultRule {
        private final java.util.List<byte[]> memoryHog = new java.util.ArrayList<>();
        @Override
        public void execute() {
            ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
            scheduler.scheduleAtFixedRate(() -> {
                // 每次分配10MB内存
                byte[] data = new byte[10 * 1024 * 1024];
                memoryHog.add(data);
                // 打印当前内存使用情况
                Runtime runtime = Runtime.getRuntime();
                long usedMemory = (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024);
                System.out.println("已使用内存: " + usedMemory + "MB");
            }, 0, 2, TimeUnit.SECONDS);
        }
    }
    /**
     * 场景3: 线程死锁故障
     */
    public static class DeadlockFault implements FaultSimulator.FaultRule {
        private final Object lock1 = new Object();
        private final Object lock2 = new Object();
        @Override
        public void execute() {
            // 线程1持有lock1，等待lock2
            new Thread(() -> {
                synchronized (lock1) {
                    System.out.println("线程1持有lock1");
                    try { Thread.sleep(500); } catch (InterruptedException e) {}
                    synchronized (lock2) {
                        System.out.println("线程1已获取lock2");
                    }
                }
            }).start();
            // 线程2持有lock2，等待lock1
            new Thread(() -> {
                synchronized (lock2) {
                    System.out.println("线程2持有lock2");
                    try { Thread.sleep(500); } catch (InterruptedException e) {}
                    synchronized (lock1) {
                        System.out.println("线程2已获取lock1");
                    }
                }
            }).start();
        }
    }
    /**
     * 场景4: 响应延迟故障
     */
    public static class LatencyFault implements FaultSimulator.FaultRule {
        private final long delayMillis;
        private final double delayProbability;
        public LatencyFault(long delayMillis, double delayProbability) {
            this.delayMillis = delayMillis;
            this.delayProbability = delayProbability;
        }
        @Override
        public void execute() {
            Random random = new Random();
            // 通过AOP或代理方式实现
            if (random.nextDouble() < delayProbability) {
                try {
                    Thread.sleep(delayMillis);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
    }
    /**
     * 场景5: 网络异常故障
     */
    public static class NetworkExceptionFault implements FaultSimulator.FaultRule {
        private final double exceptionProbability;
        public NetworkExceptionFault(double exceptionProbability) {
            this.exceptionProbability = exceptionProbability;
        }
        @Override
        public void execute() {
            Random random = new Random();
            if (random.nextDouble() < exceptionProbability) {
                throw new RuntimeException("模拟网络连接异常");
            }
        }
    }
}

业务系统模拟

package com.example.faultsimulator.business;
import java.util.concurrent.*;
import java.util.Random;
public class BusinessService {
    private final ExecutorService executorService = Executors.newFixedThreadPool(10);
    private final Random random = new Random();
    /**
     * 模拟订单处理
     */
    public String processOrder(String orderId) {
        // 模拟业务处理逻辑
        long startTime = System.currentTimeMillis();
        try {
            // 模拟数据库操作
            Thread.sleep(100 + random.nextInt(200));
            // 模拟外部接口调用
            callExternalService();
            // 模拟数据处理
            processData();
            long endTime = System.currentTimeMillis();
            return "订单 " + orderId + " 处理成功，耗时: " + (endTime - startTime) + "ms";
        } catch (Exception e) {
            return "订单 " + orderId + " 处理失败: " + e.getMessage();
        }
    }
    /**
     * 模拟高并发请求
     */
    public void simulateHighConcurrency(int concurrentClients, int requestsPerClient) {
        CountDownLatch latch = new CountDownLatch(concurrentClients);
        for (int i = 0; i < concurrentClients; i++) {
            final int clientId = i;
            executorService.submit(() -> {
                try {
                    for (int j = 0; j < requestsPerClient; j++) {
                        String orderId = "Order-" + clientId + "-" + j;
                        processOrder(orderId);
                    }
                } finally {
                    latch.countDown();
                }
            });
        }
        try {
            latch.await();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
    private void callExternalService() throws InterruptedException {
        // 模拟外部服务调用
        Thread.sleep(random.nextInt(100));
    }
    private void processData() throws InterruptedException {
        // 模拟数据处理
        Thread.sleep(random.nextInt(100));
    }
}

监控和告警系统

package com.example.faultsimulator.monitor;
import java.lang.management.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
public class SystemMonitor {
    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
    private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
    private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
    private final AtomicLong totalRequests = new AtomicLong(0);
    private final AtomicLong failedRequests = new AtomicLong(0);
    public void startMonitoring() {
        scheduler.scheduleAtFixedRate(this::monitor, 0, 5, TimeUnit.SECONDS);
    }
    private void monitor() {
        // CPU使用率
        double cpuUsage = getCpuUsage();
        // 内存使用情况
        MemoryUsage heapMemoryUsage = memoryMXBean.getHeapMemoryUsage();
        long usedMemory = heapMemoryUsage.getUsed() / (1024 * 1024);
        long maxMemory = heapMemoryUsage.getMax() / (1024 * 1024);
        // 线程数量
        int threadCount = threadMXBean.getThreadCount();
        // 死锁检测
        long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
        // 请求统计
        long total = totalRequests.get();
        long failed = failedRequests.get();
        double errorRate = total > 0 ? (double) failed / total * 100 : 0;
        // 打印监控信息
        System.out.println("========== 系统监控 ==========");
        System.out.printf("CPU使用率: %.2f%%%n", cpuUsage);
        System.out.printf("内存使用: %dMB / %dMB%n", usedMemory, maxMemory);
        System.out.println("活动线程数: " + threadCount);
        System.out.println("死锁线程: " + (deadlockedThreads != null ? deadlockedThreads.length : 0));
        System.out.printf("请求总数: %d, 失败率: %.2f%%%n", total, errorRate);
        System.out.println("================================");
    }
    private double getCpuUsage() {
        OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
        com.sun.management.OperatingSystemMXBean sunOsBean = 
            (com.sun.management.OperatingSystemMXBean) osBean;
        return sunOsBean.getProcessCpuLoad() * 100;
    }
    public void incrementRequests() {
        totalRequests.incrementAndGet();
    }
    public void incrementFailedRequests() {
        failedRequests.incrementAndGet();
    }
}

故障演练执行器

package com.example.faultsimulator;
import com.example.faultsimulator.scenarios.CommonFaultScenarios;
import com.example.faultsimulator.business.BusinessService;
import com.example.faultsimulator.monitor.SystemMonitor;
import java.util.Scanner;
import java.util.concurrent.*;
public class FaultDrillExecutor {
    private final FaultSimulator faultSimulator = FaultSimulator.getInstance();
    private final BusinessService businessService = new BusinessService();
    private final SystemMonitor systemMonitor = new SystemMonitor();
    /**
     * 执行故障演练
     */
    public void executeDrill() {
        System.out.println("====== Java故障演练系统 ======");
        System.out.println("请选择故障演练场景:");
        System.out.println("1. CPU飙升故障");
        System.out.println("2. 内存溢出故障");
        System.out.println("3. 线程死锁故障");
        System.out.println("4. 响应延迟故障");
        System.out.println("5. 网络异常故障");
        System.out.println("6. 高并发压力演练");
        System.out.println("0. 退出");
        Scanner scanner = new Scanner(System.in);
        int choice = scanner.nextInt();
        // 启动监控
        systemMonitor.startMonitoring();
        switch (choice) {
            case 1:
                cpuHignUsageDrill();
                break;
            case 2:
                memoryLeakDrill();
                break;
            case 3:
                deadlockDrill();
                break;
            case 4:
                latencyDrill();
                break;
            case 5:
                networkExceptionDrill();
                break;
            case 6:
                highConcurrencyDrill();
                break;
            case 0:
                System.out.println("退出系统");
                System.exit(0);
                break;
            default:
                System.out.println("无效选项");
        }
    }
    /**
     * CPU飙升演练
     */
    private void cpuHignUsageDrill() {
        System.out.println("开始CPU飙升故障演练...");
        // 注册故障
        CommonFaultScenarios.CPUHighUsageFault cpuFault = 
            new CommonFaultScenarios.CPUHighUsageFault();
        faultSimulator.registerFaultRule("cpu-exhaustion", cpuFault);
        // 触发故障
        faultSimulator.triggerFault("cpu-exhaustion");
        // 业务请求
        ScheduledExecutorService requestExecutor = Executors.newScheduledThreadPool(1);
        requestExecutor.scheduleAtFixedRate(() -> {
            SystemMonitor monitor = getMonitor();
            businessService.processOrder("CPU-Fault-Order");
        }, 0, 1, TimeUnit.SECONDS);
        // 30秒后自动恢复
        ScheduledExecutorService recoveryExecutor = Executors.newScheduledThreadPool(1);
        recoveryExecutor.schedule(() -> {
            cpuFault.stop();
            System.out.println("CPU故障演练完成，系统状态恢复");
        }, 30, TimeUnit.SECONDS);
    }
    /**
     * 内存溢出演练
     */
    private void memoryLeakDrill() {
        System.out.println("开始内存溢出故障演练...");
        CommonFaultScenarios.MemoryLeakFault memoryFault = 
            new CommonFaultScenarios.MemoryLeakFault();
        // 设置JVM参数
        System.setProperty("JAVA_OPTS", "-Xmx256m");
        // 触发故障
        Thread faultThread = new Thread(() -> {
            memoryFault.execute();
            // 定时检查内存
            while (true) {
                Runtime runtime = Runtime.getRuntime();
                long usedMemory = (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024);
                long maxMemory = runtime.maxMemory() / (1024 * 1024);
                if (usedMemory > maxMemory * 0.8) {
                    System.out.println("警告: 内存使用超过80%，即将触发OOM");
                    throw new OutOfMemoryError("模拟内存溢出故障");
                }
                try {
                    Thread.sleep(5000);
                } catch (InterruptedException e) {
                    break;
                }
            }
        });
        faultThread.start();
    }
    /**
     * 死锁演练
     */
    private void deadlockDrill() {
        System.out.println("开始线程死锁故障演练...");
        CommonFaultScenarios.DeadlockFault deadlockFault = 
            new CommonFaultScenarios.DeadlockFault();
        faultSimulator.registerFaultRule("deadlock", deadlockFault);
        faultSimulator.triggerFault("deadlock");
        // 等待30秒后检查死锁并恢复
        ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
        scheduler.schedule(() -> {
            ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
            long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
            if (deadlockedThreads != null) {
                System.out.println("检测到 " + deadlockedThreads.length + " 个死锁线程");
                // 中断死锁线程
                for (long threadId : deadlockedThreads) {
                    ThreadInfo threadInfo = threadMXBean.getThreadInfo(threadId);
                    System.out.println("死锁线程: " + threadInfo.getThreadName());
                    Thread thread = ThreadUtils.findThread(threadId);
                    if (thread != null) {
                        thread.interrupt();
                    }
                }
            } else {
                System.out.println("未检测到死锁");
            }
        }, 30, TimeUnit.SECONDS);
    }
    /**
     * 延迟演练
     */
    private void latencyDrill() {
        System.out.println("开始响应延迟故障演练...");
        long delay = 3000; // 3秒延迟
        double probability = 0.5; // 50%概率
        CommonFaultScenarios.LatencyFault latencyFault = 
            new CommonFaultScenarios.LatencyFault(delay, probability);
        faultSimulator.registerFaultRule("latency", latencyFault);
        // 发送10个测试请求
        for (int i = 0; i < 10; i++) {
            faultSimulator.triggerFault("latency");
            long start = System.currentTimeMillis();
            String result = businessService.processOrder("Latency-Order-" + i);
            long end = System.currentTimeMillis();
            System.out.println("请求" + i + ": " + result + ", 实际耗时: " + (end - start) + "ms");
        }
    }
    /**
     * 网络异常演练
     */
    private void networkExceptionDrill() {
        System.out.println("开始网络异常故障演练...");
        double probability = 0.3; // 30%概率抛出异常
        CommonFaultScenarios.NetworkExceptionFault networkFault = 
            new CommonFaultScenarios.NetworkExceptionFault(probability);
        // 模拟网络异常
        for (int i = 0; i < 10; i++) {
            try {
                networkFault.execute();
                System.out.println("请求" + i + ": 网络正常");
            } catch (Exception e) {
                System.out.println("请求" + i + ": 网络异常 - " + e.getMessage());
                systemMonitor.incrementFailedRequests();
            }
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                break;
            }
        }
    }
    /**
     * 高并发演练
     */
    private void highConcurrencyDrill() {
        System.out.println("开始高并发压力演练...");
        int concurrentClients = 100;
        int requestsPerClient = 50;
        ScheduledExecutorService monitorExecutor = Executors.newScheduledThreadPool(1);
        monitorExecutor.scheduleAtFixedRate(() -> {
            System.out.println("持续高并发压力测试中...");
        }, 0, 5, TimeUnit.SECONDS);
        businessService.simulateHighConcurrency(concurrentClients, requestsPerClient);
        System.out.println("高并发演练完成");
    }
    private SystemMonitor getMonitor() {
        return systemMonitor;
    }
    public static void main(String[] args) throws InterruptedException {
        FaultDrillExecutor executor = new FaultDrillExecutor();
        // 添加钩子，确保程序退出时清理资源
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            System.out.println("\n清理资源...");
            FaultSimulator.getInstance().disableAllFaults();
        }));
        // 运行演练
        executor.executeDrill();
        // 等待演练完成
        Thread.sleep(35000); // 等待35秒
        System.out.println("\n演练结束，查看监控报告...");
    }
}
// 线程工具类
class ThreadUtils {
    public static Thread findThread(long threadId) {
        for (Thread thread : Thread.getAllStackTraces().keySet()) {
            if (thread.getId() == threadId) {
                return thread;
            }
        }
        return null;
    }
}

故障演练结果分析

package com.example.faultsimulator.analysis;
import java.util.*;
import java.util.concurrent.*;
public class DrillAnalyzer {
    private final List<DrillResult> results = new CopyOnWriteArrayList<>();
    public void addResult(DrillResult result) {
        results.add(result);
    }
    /**
     * 生成分析报告
     */
    public void generateReport() {
        long totalDuration = 0;
        int successCount = 0;
        int failedCount = 0;
        System.out.println("\n===== 故障演练分析报告 =====");
        System.out.println("总演练次数: " + results.size());
        for (DrillResult result : results) {
            totalDuration += result.getDurationMs();
            if (result.isSuccess()) {
                successCount++;
            } else {
                failedCount++;
            }
            System.out.println("演练场景: " + result.getScenario());
            System.out.println("执行时间: " + result.getDurationMs() + "ms");
            System.out.println("成功率: " + result.getSuccessRate() + "%");
            System.out.println("---");
        }
        System.out.printf("平均耗时: %.2fms%n", totalDuration / (double) results.size());
        System.out.println("成功次数: " + successCount);
        System.out.println("失败次数: " + failedCount);
        System.out.printf("总体成功率: %.2f%%%n", (successCount / (double) results.size()) * 100);
    }
    public static class DrillResult {
        private final String scenario;
        private final long durationMs;
        private final boolean success;
        private final double successRate;
        public DrillResult(String scenario, long durationMs, boolean success, double successRate) {
            this.scenario = scenario;
            this.durationMs = durationMs;
            this.success = success;
            this.successRate = successRate;
        }
        public long getDurationMs() { return durationMs; }
        public boolean isSuccess() { return success; }
        public double getSuccessRate() { return successRate; }
        public String getScenario() { return scenario; }
    }
}

使用说明

运行准备

<project>
    <dependencies>
        <!-- JVM监控依赖 -->
        <dependency>
            <groupId>com.sun</groupId>
            <artifactId>tools</artifactId>
            <version>1.8</version>
        </dependency>
    </dependencies>
</project>

启动命令

# 内存溢出演练时添加JVM参数
java -Xmx256m -Xms128m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs/heap.hprof FaultDrillExecutor
# 常规启动
java FaultDrillExecutor

安全建议

1. 生产环境限制：故障演练应在测试环境或预生产环境执行
2. 自动恢复机制：所有故障规则均应设置自动恢复时间
3. 监控告警：演练过程中必须启用实时监控
4. 演练审计：记录所有演练操作，便于回溯分析
5. 优雅降级：确保系统具有降级和容错机制

这个故障演练框架能够帮助团队：

• 验证系统的容错性
• 测试故障恢复机制
• 评估系统的高可用性
• 培训运维人员的应急处理能力