游戏 DevOps 最佳实践：构建高效研发运维体系

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# game-server/Dockerfile
FROM golang:1.21-alpine AS builder

# 安装依赖
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download

# 编译
COPY . .
RUN CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags="-w -s" -o game-server

# 运行时镜像
FROM alpine:latest

RUN apk --no-cache add ca-certificates tzdata

WORKDIR /root/

# 复制二进制文件
COPY --from=builder /app/game-server .

# 设置时区
ENV TZ=Asia/Shanghai

# 暴露端口
EXPOSE 8080 8443

# 健康检查
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
  CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1

# 运行服务
CMD ["./game-server"]

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

# kubernetes/game-server-deployment.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: game-server-config
data:
  LOG_LEVEL: "info"
  DB_HOST: "postgres.default.svc.cluster.local"
  DB_PORT: "5432"
  REDIS_HOST: "redis.default.svc.cluster.local"
  REDIS_PORT: "6379"

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: game-server
  labels:
    app: game-server
spec:
  replicas: 3
  selector:
    matchLabels:
      app: game-server
  template:
    metadata:
      labels:
        app: game-server
        version: v1.0.0
    spec:
      containers:
      - name: game-server
        image: registry.example.com/game-server:v1.0.0
        ports:
        - name: http
          containerPort: 8080
          protocol: TCP
        - name: grpc
          containerPort: 8443
          protocol: TCP
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        envFrom:
        - configMapRef:
            name: game-server-config
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 3
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 2
          failureThreshold: 2
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]
---
apiVersion: v1
kind: Service
metadata:
  name: game-server
spec:
  type: ClusterIP
  ports:
  - name: http
    port: 80
    targetPort: 8080
  - name: grpc
    port: 443
    targetPort: 8443
  selector:
    app: game-server
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: game-server-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: game-server
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  - type: Pods
    pods:
      metric:
        name: active_connections
      target:
        type: AverageValue
        averageValue: "1000"
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 50
        periodSeconds: 60
      - type: Pods
        value: 5
        periodSeconds: 60
    scaleUp:
      stabilizationWindowSeconds: 0
      policies:
      - type: Percent
        value: 100
        periodSeconds: 30
      - type: Pods
        value: 10
        periodSeconds: 30

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

# .github/workflows/release-client.yml
name: Release Game Client

on:
  push:
    tags:
      - 'v*'

jobs:
  build-android:
    name: Build Android
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          lfs: true

      - name: Cache Unity Library
        uses: actions/cache@v3
        with:
          path: Library
          key: Library-android-${{ hashFiles('Assets/**', 'Packages/**') }}

      - name: Build Android APK
        uses: game-ci/unity-builder@v4
        with:
          targetPlatform: Android
          buildName: Game
          buildMethod: UnityBuilder.Build
          androidExportType: androidPackage
          androidKeyaliasName: release
          androidKeystoreName: user.keystore
          androidKeystorePass: ${{ secrets.KEYSTORE_PASSWORD }}
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}

      - name: Upload to Play Store
        uses: r0adkll/upload-google-play@v1
        with:
          serviceAccountJsonPlainText: ${{ secrets.GOOGLE_PLAY_SERVICE_ACCOUNT }}
          packageName: com.company.game
          releaseFiles: build/Android/*.apk
          track: production
          status: completed

  build-ios:
    name: Build iOS
    runs-on: macos-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          lfs: true

      - name: Build iOS
        uses: game-ci/unity-builder@v4
        with:
          targetPlatform: iOS
          buildMethod: UnityBuilder.Build
          iosVersion: 1
        env:
          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}

      - name: Upload to TestFlight
        uses: apple-actions/upload-testflight-build@v1
        with:
          app-type: ios
          app-specific-password: ${{ secrets.APPLE_SPECIFIC_PASSWORD }}
          apple-id: ${{ secrets.APPLE_ID }}
          password: ${{ secrets.APPLE_PASSWORD }}
          file-path: build/iOS/*.ipa

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111

// services/monitoring/GameMetricsCollector.ts
class GameMetricsCollector {
    private prometheus: PrometheusRegistry;

    // 定义指标
    private metrics = {
        // 在线玩家数
        onlinePlayers: new Gauge({
            name: 'game_online_players',
            help: 'Number of online players',
            labelNames: ['server', 'region']
        }),

        // 游戏帧率
        fps: new Histogram({
            name: 'game_fps',
            help: 'Game frames per second',
            labelNames: ['server', 'scene'],
            buckets: [30, 45, 55, 60]
        }),

        // 匹配延迟
        matchLatency: new Histogram({
            name: 'game_match_latency_ms',
            help: 'Matchmaking latency in milliseconds',
            labelNames: ['region'],
            buckets: [50, 100, 200, 500, 1000, 2000]
        }),

        // 错误率
        errors: new Counter({
            name: 'game_errors_total',
            help: 'Total number of errors',
            labelNames: ['server', 'error_type', 'severity']
        }),

        // API 请求
        apiRequests: new Counter({
            name: 'game_api_requests_total',
            help: 'Total number of API requests',
            labelNames: ['method', 'endpoint', 'status']
        }),

        // 活跃玩家
        activePlayers: new Gauge({
            name: 'game_active_players',
            help: 'Number of active players in last 24h',
            labelNames: ['platform']
        }),

        // 收入
        revenue: new Counter({
            name: 'game_revenue_total',
            help: 'Total revenue',
            labelNames: ['currency', 'item_type']
        }),

        // 服务器负载
        serverLoad: new Gauge({
            name: 'game_server_load',
            help: 'Server load percentage',
            labelNames: ['server', 'type'] // cpu, memory, network
        })
    };

    // 收集在线玩家数
    async collectOnlinePlayers(servers: ServerInfo[]): Promise<void> {
        for (const server of servers) {
            const count = await this.getServerOnlinePlayerCount(server);

            this.metrics.onlinePlayers.set(
                { server: server.name, region: server.region },
                count
            );
        }
    }

    // 收集帧率
    async collectFPS(server: ServerInfo): Promise<void> {
        const fpsData = await this.getServerFPS(server);

        for (const [scene, fps] of Object.entries(fpsData)) {
            this.metrics.fps.observe(
                { server: server.name, scene },
                fps
            );
        }
    }

    // 收集API指标
    recordAPICall(method: string, endpoint: string, statusCode: number): void {
        this.metrics.apiRequests.inc(
            { method, endpoint, status: statusCode.toString() }
        );
    }

    // 收集收入
    recordRevenue(currency: string, itemType: string, amount: number): void {
        this.metrics.revenue.inc(
            { currency, item_type: itemType },
            amount
        );
    }

    // 收集错误
    recordError(server: string, errorType: string, severity: string): void {
        this.metrics.errors.inc(
            { server, error_type, severity }
        );
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

// services/logging/LoggingService.ts
class GameLoggingService {
    private transports: LogTransport[] = [];

    constructor() {
        // 控制台输出
        this.addTransport(new ConsoleTransport());

        // 文件输出
        this.addTransport(new FileTransport({
            filename: '/var/log/game/game.log',
            maxSize: '100M',
            maxFiles: 10
        }));

        // 日志服务
        this.addTransport(new ElasticsearchTransport({
            node: 'http://elasticsearch:9200',
            index: 'game-logs'
        }));
    }

    // 记录游戏事件
    logGameEvent(event: GameEvent): void {
        this.log('info', 'game_event', {
            event_type: event.type,
            player_id: event.playerId,
            server_id: event.serverId,
            data: event.data,
            timestamp: Date.now()
        });
    }

    // 记录玩家行为
    logPlayerAction(action: PlayerAction): void {
        this.log('info', 'player_action', {
            player_id: action.playerId,
            action_type: action.type,
            target: action.target,
            result: action.result,
            timestamp: Date.now()
        });
    }

    // 记录错误
    logError(error: GameError): void {
        this.log('error', 'game_error', {
            error_type: error.type,
            error_message: error.message,
            stack_trace: error.stack,
            player_id: error.playerId,
            server_id: error.serverId,
            context: error.context,
            timestamp: Date.now()
        });
    }

    // 记录性能指标
    logPerformance(metrics: PerformanceMetrics): void {
        this.log('debug', 'performance', {
            server_id: metrics.serverId,
            fps: metrics.fps,
            ping: metrics.ping,
            memory: metrics.memory,
            cpu: metrics.cpu,
            players: metrics.players,
            timestamp: Date.now()
        });
    }

    private log(level: string, category: string, data: any): void {
        const logEntry: LogEntry = {
            level,
            category,
            data,
            timestamp: new Date().toISOString(),
            hostname: os.hostname(),
            pid: process.pid
        };

        for (const transport of this.transports) {
            transport.write(logEntry);
        }
    }
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

// services/scaling/GameAutoScaler.ts
class GameAutoScaler {
    private k8s: KubernetesClient;
    private rules: ScalingRule[] = [];

    // 扩缩容评估
    async evaluateScaling(): Promise<ScalingDecision[]> {
        const decisions: ScalingDecision[] = [];

        for (const rule of this.rules) {
            const decision = await this.evaluateRule(rule);
            if (decision) {
                decisions.push(decision);
            }
        }

        return decisions;
    }

    private async evaluateRule(rule: ScalingRule): Promise<ScalingDecision | null> {
        // 获取当前指标
        const currentValue = await this.getMetricValue(rule.metric);

        // 判断是否需要扩容
        if (currentValue > rule.scaleUpThreshold) {
            return {
                type: 'scale_up',
                deployment: rule.deployment,
                currentReplicas: await this.getCurrentReplicas(rule.deployment),
                desiredReplicas: this.calculateDesiredReplicas(rule, currentValue, 'up'),
                reason: `${rule.metric} (${currentValue}) > ${rule.scaleUpThreshold}`,
                metric: rule.metric,
                value: currentValue
            };
        }

        // 判断是否需要缩容
        if (currentValue < rule.scaleDownThreshold) {
            return {
                type: 'scale_down',
                deployment: rule.deployment,
                currentReplicas: await this.getCurrentReplicas(rule.deployment),
                desiredReplicas: this.calculateDesiredReplicas(rule, currentValue, 'down'),
                reason: `${rule.metric} (${currentValue}) < ${rule.scaleDownThreshold}`,
                metric: rule.metric,
                value: currentValue
            };
        }

        return null;
    }

    private calculateDesiredReplicas(
        rule: ScalingRule,
        currentValue: number,
        direction: 'up' | 'down'
    ): number {
        const current = await this.getCurrentReplicas(rule.deployment);

        if (direction === 'up') {
            // 扩容：基于阈值计算
            const ratio = currentValue / rule.scaleUpThreshold;
            const desired = Math.ceil(current * ratio);

            return Math.min(desired, rule.maxReplicas);
        } else {
            // 缩容：基于阈值计算
            const ratio = rule.scaleDownThreshold / currentValue;
            const desired = Math.floor(current * ratio);

            return Math.max(desired, rule.minReplicas);
        }
    }

    // 内置扩缩容规则
    getBuiltinRules(): ScalingRule[] {
        return [
            {
                id: 'player_count',
                deployment: 'game-server',
                metric: 'active_players',
                scaleUpThreshold: 800,
                scaleDownThreshold: 200,
                minReplicas: 3,
                maxReplicas: 20
            },
            {
                id: 'cpu_usage',
                deployment: 'game-server',
                metric: 'cpu_usage_percent',
                scaleUpThreshold: 70,
                scaleDownThreshold: 30,
                minReplicas: 3,
                maxReplicas: 20
            },
            {
                id: 'memory_usage',
                deployment: 'game-server',
                metric: 'memory_usage_percent',
                scaleUpThreshold: 80,
                scaleDownThreshold: 40,
                minReplicas: 3,
                maxReplicas: 20
            },
            {
                id: 'queue_depth',
                deployment: 'matchmaker',
                metric: 'match_queue_depth',
                scaleUpThreshold: 1000,
                scaleDownThreshold: 100,
                minReplicas: 2,
                maxReplicas: 10
            }
        ];
    }
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

// services/recovery/FaultRecoveryService.ts
class FaultRecoveryService {
    private detectors: FaultDetector[] = [];
    private recoveryActions = new Map<string, RecoveryAction>();

    // 故障检测
    async detectFaults(): Promise<Fault[]> {
        const faults: Fault[] = [];

        for (const detector of this.detectors) {
            const detectedFaults = await detector.detect();
            faults.push(...detectedFaults);
        }

        return faults;
    }

    // 故障恢复
    async recoverFault(fault: Fault): Promise<RecoveryResult> {
        console.log(`Recovering from fault: ${fault.type}`);

        // 获取恢复动作
        const action = this.recoveryActions.get(fault.type);

        if (!action) {
            return {
                fault,
                success: false,
                message: `No recovery action for ${fault.type}`
            };
        }

        try {
            // 执行恢复动作
            await action.execute(fault);

            // 验证恢复
            const recovered = await this.verifyRecovery(fault);

            return {
                fault,
                success: recovered,
                message: recovered ? 'Recovery successful' : 'Recovery verification failed'
            };

        } catch (error) {
            return {
                fault,
                success: false,
                message: `Recovery failed: ${error.message}`
            };
        }
    }

    // 内置故障检测器
    getBuiltinDetectors(): FaultDetector[] {
        return [
            // Pod 崩溃检测
            {
                name: 'pod_crash',
                detect: async () => {
                    const pods = await this.k8s.core.listPodForAllNamespaces();

                    return pods
                        .filter(pod =>
                            pod.status.phase === 'CrashLoopBackOff' ||
                            parseInt(pod.status.containerStatuses[0].restartCount) > 5
                        )
                        .map(pod => ({
                            type: 'pod_crash',
                            severity: 'high',
                            resource: `Pod/${pod.metadata.name}`,
                            message: `Pod ${pod.metadata.name} is crashing`,
                            data: { pod }
                        }));
                }
            },

            // 高延迟检测
            {
                name: 'high_latency',
                detect: async () => {
                    const latency = await this.getAverageLatency();

                    if (latency > 500) {
                        return [{
                            type: 'high_latency',
                            severity: 'medium',
                            resource: 'game-server',
                            message: `High latency detected: ${latency}ms`,
                            data: { latency }
                        }];
                    }

                    return [];
                }
            },

            // 内存泄漏检测
            {
                name: 'memory_leak',
                detect: async () => {
                    const pods = await this.k8s.core.listPodForAllNamespaces();
                    const results: Fault[] = [];

                    for (const pod of pods) {
                        const containers = pod.status.containerStatuses || [];

                        for (const container of containers) {
                            const memoryUsage = container.state?.running?.startedAt;

                            if (memoryUsage && memoryUsage > 500 * 1024 * 1024) { // 500MB
                                results.push({
                                    type: 'memory_leak',
                                    severity: 'high',
                                    resource: `Pod/${pod.metadata.name}/${container.name}`,
                                    message: `Possible memory leak: ${memoryUsage / 1024 / 1024}MB`,
                                    data: { pod, container }
                                });
                            }
                        }
                    }

                    return results;
                }
            },

            // 连接数检测
            {
                name: 'connection_exhaustion',
                detect: async () => {
                    const connections = await this.getActiveConnections();

                    if (connections > 9000) {
                        return [{
                            type: 'connection_exhaustion',
                            severity: 'critical',
                            resource: 'database',
                            message: `Connection pool exhausted: ${connections}/10000`,
                            data: { connections }
                        }];
                    }

                    return [];
                }
            }
        ];
    }
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104

// services/backup/BackupManager.ts
class GameBackupManager {
    // 游戏数据备份
    async backupGameData(options: BackupOptions): Promise<BackupResult> {
        const backupId = this.generateBackupId();
        const timestamp = new Date();

        try {
            // 数据库备份
            const dbBackup = await this.backupDatabase({
                type: 'postgres',
                databases: ['game_db', 'chat_db'],
                backupId
            });

            // Redis 备份
            const redisBackup = await this.backupRedis({
                nodes: ['redis-master', 'redis-slave'],
                backupId
            });

            // 对象存储备份
            const assetsBackup = await this.backupAssets({
                buckets: ['game-assets', 'user-uploads'],
                backupId
            });

            // 配置备份
            const configBackup = await this.backupConfig({
                services: ['game-server', 'matchmaker', 'chat'],
                backupId
            });

            const result: BackupResult = {
                backupId,
                timestamp: timestamp.toISOString(),
                components: {
                    database: dbBackup,
                    redis: redisBackup,
                    assets: assetsBackup,
                    config: configBackup
                },
                status: 'completed'
            };

            // 保存备份元数据
            await this.saveBackupMetadata(result);

            return result;

        } catch (error) {
            return {
                backupId,
                timestamp: timestamp.toISOString(),
                status: 'failed',
                error: error.message
            };
        }
    }

    // 数据恢复
    async restoreGameData(backupId: string, targetEnv: string): Promise<RestoreResult> {
        // 获取备份元数据
        const metadata = await this.getBackupMetadata(backupId);

        if (!metadata) {
            throw new Error(`Backup not found: ${backupId}`);
        }

        const results: ComponentRestoreResult[] = [];

        // 恢复数据库
        const dbRestore = await this.restoreDatabase({
            backupId,
            targetEnv,
            component: metadata.components.database
        });
        results.push(dbRestore);

        // 恢复 Redis
        const redisRestore = await this.restoreRedis({
            backupId,
            targetEnv,
            component: metadata.components.redis
        });
        results.push(redisRestore);

        // 恢复配置
        const configRestore = await this.restoreConfig({
            backupId,
            targetEnv,
            component: metadata.components.config
        });
        results.push(configRestore);

        return {
            backupId,
            targetEnv,
            timestamp: new Date().toISOString(),
            components: results,
            status: results.every(r => r.success) ? 'completed' : 'partial'
        };
    }
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

// services/drill/DisasterDrillService.ts
class DisasterDrillService {
    // 执行灾难演练
    async executeDrill(scenario: DrillScenario): Promise<DrillResult> {
        console.log(`Starting disaster drill: ${scenario.name}`);

        const startTime = Date.now();

        try {
            // 备份当前状态
            const backupId = await this.createPreDrillBackup();

            // 注入故障
            await this.injectFault(scenario.fault);

            // 验证系统响应
            const detectionTime = await this.measureFaultDetection(scenario);

            // 执行恢复
            const recoveryTime = await this.performRecovery(scenario);

            // 验证恢复结果
            const verification = await this.verifyRecovery(scenario);

            // 恢复原状态
            await this.restoreFromBackup(backupId);

            const duration = Date.now() - startTime;

            return {
                scenarioId: scenario.id,
                status: 'completed',
                duration,
                detectionTime,
                recoveryTime,
                verification,
                timestamp: new Date().toISOString()
            };

        } catch (error) {
            // 尝试恢复
            await this.emergencyRestore();

            return {
                scenarioId: scenario.id,
                status: 'failed',
                error: error.message,
                timestamp: new Date().toISOString()
            };
        }
    }

    // 演练场景
    getBuiltinScenarios(): DrillScenario[] {
        return [
            {
                id: 'db_failover',
                name: '数据库故障转移',
                description: '模拟主数据库故障，验证自动故障转移',
                fault: {
                    type: 'database_down',
                    target: 'postgres-primary'
                },
                expectations: {
                    detectionTime: 30000, // 30秒内检测到
                    recoveryTime: 120000, // 2分钟内恢复
                    dataLoss: false
                }
            },
            {
                id: 'server_crash',
                name: '游戏服务器崩溃',
                description: '模拟游戏服务器进程崩溃',
                fault: {
                    type: 'process_kill',
                    target: 'game-server-0'
                },
                expectations: {
                    detectionTime: 10000, // 10秒内检测到
                    recoveryTime: 60000, // 1分钟内重启
                    playerReconnect: true
                }
            },
            {
                id: 'network_partition',
                name: '网络分区',
                description: '模拟网络分区，验证服务降级',
                fault: {
                    type: 'network_partition',
                    affectedNodes: ['game-server-1', 'game-server-2']
                },
                expectations: {
                    detectionTime: 15000,
                    degradedMode: true,
                    recoveryTime: 180000
                }
            },
            {
                id: 'ddos_attack',
                name: 'DDoS 攻击',
                description: '模拟 DDoS 攻击，验证防护措施',
                fault: {
                    type: 'traffic_spike',
                    target: 'loadbalancer',
                    multiplier: 10
                },
                expectations: {
                    detectionTime: 5000,
                    mitigationTime: 30000,
                    serviceContinuity: true
                }
            }
        ];
    }
}