/** * Fleet Deployment Service - 실제 배포 엔진 * - 릴리즈를 선택한 디바이스에 배포 * - 롤아웃 전략: immediate, canary, rolling * - 각 디바이스별 deploy 커맨드 발행 (MQTT) * - 응답 받아 성공/실패 집계 */ import { query, queryOne } from "../database/db"; import { logger } from "../utils/logger"; import { FleetCommandService } from "./fleetCommandService"; export type DeploymentStatus = | "pending" // 생성됨, 아직 실행 전 | "running" // 진행 중 | "paused" // 일시정지 (실패율 초과) | "completed" // 완료 | "failed" // 실패 | "cancelled" // 취소됨 | "rolled_back"; export interface FleetDeployment { id?: number; release_id: number; target_type: "all" | "company" | "group" | "device_list"; target_value?: string; // company_code, group name, or device ids (csv) rollout_strategy: "immediate" | "canary" | "rolling"; rollout_percentage?: number; batch_size?: number; max_failures?: number; pause_on_failure?: boolean; description?: string; scheduled_at?: Date; status?: DeploymentStatus; } export class FleetDeploymentService { static async list(filter: { status?: string; release_id?: number } = {}) { const wheres: string[] = []; const params: any[] = []; let idx = 1; if (filter.status) { wheres.push(`d.status = $${idx++}`); params.push(filter.status); } if (filter.release_id) { wheres.push(`d.release_id = $${idx++}`); params.push(filter.release_id); } const where = wheres.length ? `WHERE ${wheres.join(" AND ")}` : ""; return await query( `SELECT d.*, r.version as release_version, r.backend_image, r.frontend_image, r.agent_image FROM fleet_deployments d LEFT JOIN fleet_releases r ON d.release_id = r.id ${where} ORDER BY d.created_at DESC LIMIT 100`, params, ); } static async get(id: number) { return await queryOne( `SELECT d.*, r.version as release_version, r.backend_image, r.frontend_image, r.agent_image FROM fleet_deployments d LEFT JOIN fleet_releases r ON d.release_id = r.id WHERE d.id = $1`, [id], ); } static async getStatus(deploymentId: number) { return await query( `SELECT ds.*, d.device_name, d.company_code FROM fleet_deployment_status ds LEFT JOIN fleet_devices d ON ds.device_id = d.device_id WHERE ds.deployment_id = $1 ORDER BY ds.device_id`, [deploymentId], ); } static async create(data: FleetDeployment & { created_by?: string }): Promise { if (!data.release_id || !data.target_type) { throw new Error("release_id, target_type 필수"); } // 대상 디바이스 선정 const deviceIds = await this.resolveTargetDevices(data.target_type, data.target_value); if (deviceIds.length === 0) throw new Error("대상 디바이스가 없습니다."); const deploy = await query( `INSERT INTO fleet_deployments (release_id, target_type, target_value, rollout_strategy, rollout_percentage, batch_size, max_failures, pause_on_failure, description, scheduled_at, status, total_devices, created_by) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,'pending',$11,$12) RETURNING *`, [ data.release_id, data.target_type, data.target_value || null, data.rollout_strategy || "rolling", data.rollout_percentage || 100, data.batch_size || 10, data.max_failures || 3, data.pause_on_failure !== false, data.description || null, data.scheduled_at || null, deviceIds.length, data.created_by || null, ], ); // 각 대상 디바이스별 status 레코드 생성 for (const did of deviceIds) { await query( `INSERT INTO fleet_deployment_status (deployment_id, device_id, status) VALUES ($1, $2, 'pending')`, [deploy[0].id, did], ); } logger.info(`[Fleet Deploy] 생성: id=${deploy[0].id}, 대상 ${deviceIds.length}개`); return { ...deploy[0], device_count: deviceIds.length }; } /** * 배포 대상 디바이스 목록 해석 */ static async resolveTargetDevices( targetType: string, targetValue?: string | null, ): Promise { if (targetType === "all") { const r = await query( `SELECT device_id FROM fleet_devices WHERE is_online = TRUE`, ); return r.map((x: any) => x.device_id); } if (targetType === "company" && targetValue) { const r = await query( `SELECT device_id FROM fleet_devices WHERE company_code = $1 AND is_online = TRUE`, [targetValue], ); return r.map((x: any) => x.device_id); } if (targetType === "group" && targetValue) { const r = await query( `SELECT device_id FROM fleet_devices WHERE device_group = $1 AND is_online = TRUE`, [targetValue], ); return r.map((x: any) => x.device_id); } if (targetType === "device_list" && targetValue) { return targetValue.split(",").map((s) => s.trim()).filter(Boolean); } return []; } /** * 배포 시작 (실제 실행) * - 전략별 배치 처리 * - 각 디바이스에 deploy 커맨드 발행 (MQTT) * - 실패율 모니터링 */ static async start(deploymentId: number, issuedBy?: string): Promise { const deploy = await this.get(deploymentId); if (!deploy) throw new Error("배포 없음"); if (deploy.status !== "pending" && deploy.status !== "paused") { throw new Error(`현재 상태(${deploy.status})에서 시작 불가`); } await query( `UPDATE fleet_deployments SET status = 'running', started_at = COALESCE(started_at, NOW()) WHERE id = $1`, [deploymentId], ); // 대기 중 디바이스 조회 const pendingDevices = await query( `SELECT device_id FROM fleet_deployment_status WHERE deployment_id = $1 AND status = 'pending' ORDER BY device_id`, [deploymentId], ); logger.info(`[Fleet Deploy] 시작: id=${deploymentId}, 대기중 ${pendingDevices.length}개`); // 전략별 실행 const strategy = deploy.rollout_strategy; const batchSize = deploy.batch_size || 10; // 비동기로 백그라운드 배포 진행 this.runDeployment(deploymentId, pendingDevices.map((d: any) => d.device_id), strategy, batchSize, issuedBy) .catch((e) => { logger.error(`[Fleet Deploy] 실행 에러 (id=${deploymentId}):`, e); }); return { deploymentId, status: "running", scheduled: pendingDevices.length }; } /** * 실제 배포 루프 (백그라운드) */ private static async runDeployment( deploymentId: number, deviceIds: string[], strategy: string, batchSize: number, issuedBy?: string, ): Promise { const deploy = await this.get(deploymentId); const release = deploy ? { image: deploy.backend_image, version: deploy.release_version } : {}; let failures = 0; const maxFailures = deploy?.max_failures || 3; const pauseOnFail = deploy?.pause_on_failure !== false; // 카나리: 첫 1개만 먼저 배포 const executeOrder: string[][] = []; if (strategy === "canary" && deviceIds.length > 1) { executeOrder.push([deviceIds[0]]); // canary // 나머지는 배치로 for (let i = 1; i < deviceIds.length; i += batchSize) { executeOrder.push(deviceIds.slice(i, i + batchSize)); } } else if (strategy === "rolling") { for (let i = 0; i < deviceIds.length; i += batchSize) { executeOrder.push(deviceIds.slice(i, i + batchSize)); } } else { // immediate executeOrder.push(deviceIds); } for (const batch of executeOrder) { // 취소 체크 const cur = await this.get(deploymentId); if (!cur || cur.status === "cancelled") { logger.info(`[Fleet Deploy] 취소 감지: id=${deploymentId}`); return; } await Promise.all( batch.map((did) => this.deployToDevice(deploymentId, did, deploy, issuedBy)), ); // 배치 완료 후 실패율 체크 const failCount = await queryOne( `SELECT COUNT(*) as n FROM fleet_deployment_status WHERE deployment_id = $1 AND status IN ('failed','timeout')`, [deploymentId], ); failures = parseInt(failCount?.n || 0); if (pauseOnFail && failures >= maxFailures) { await query( `UPDATE fleet_deployments SET status = 'paused', completed_at = NULL WHERE id = $1`, [deploymentId], ); logger.warn(`[Fleet Deploy] 실패율 초과로 일시정지: id=${deploymentId} (실패 ${failures}개)`); return; } // 카나리: 첫 배치 완료 후 안정성 대기 if (strategy === "canary" && executeOrder.indexOf(batch) === 0) { await new Promise((r) => setTimeout(r, 3000)); // 3초 관찰 } } // 최종 집계 const stats = await queryOne( `SELECT COUNT(*) FILTER (WHERE status = 'success') as success, COUNT(*) FILTER (WHERE status IN ('failed','timeout')) as failed, COUNT(*) as total FROM fleet_deployment_status WHERE deployment_id = $1`, [deploymentId], ); const finalStatus = parseInt(stats.failed) > 0 ? (parseInt(stats.success) > 0 ? "completed" : "failed") : "completed"; await query( `UPDATE fleet_deployments SET status = $2, completed_at = NOW(), success_count = $3, failed_count = $4 WHERE id = $1`, [deploymentId, finalStatus, parseInt(stats.success), parseInt(stats.failed)], ); logger.info( `[Fleet Deploy] 완료: id=${deploymentId} 성공 ${stats.success} / 실패 ${stats.failed} / 총 ${stats.total}`, ); } /** * 개별 디바이스 배포 (MQTT 커맨드) */ private static async deployToDevice( deploymentId: number, deviceId: string, deploy: any, issuedBy?: string, ): Promise { try { await query( `UPDATE fleet_deployment_status SET status = 'running', started_at = NOW() WHERE deployment_id = $1 AND device_id = $2`, [deploymentId, deviceId], ); // MQTT로 deploy 커맨드 발행 const cmd = await FleetCommandService.issueCommand( deviceId, "deploy", { deployment_id: deploymentId, release_id: deploy.release_id, backend_image: deploy.backend_image, frontend_image: deploy.frontend_image, agent_image: deploy.agent_image, version: deploy.release_version, }, issuedBy, 600, // 10분 타임아웃 ); // 응답 대기 (폴링) - 60초 for (let i = 0; i < 60; i++) { await new Promise((r) => setTimeout(r, 1000)); const cs = await queryOne( `SELECT status FROM fleet_commands WHERE id = $1`, [cmd.id], ); if (["success", "failed", "timeout"].includes(cs?.status)) { await query( `UPDATE fleet_deployment_status SET status = $3, completed_at = NOW() WHERE deployment_id = $1 AND device_id = $2`, [deploymentId, deviceId, cs.status], ); return; } } // 타임아웃 await query( `UPDATE fleet_deployment_status SET status = 'timeout', completed_at = NOW(), error_message = '응답 타임아웃' WHERE deployment_id = $1 AND device_id = $2`, [deploymentId, deviceId], ); } catch (e: any) { await query( `UPDATE fleet_deployment_status SET status = 'failed', completed_at = NOW(), error_message = $3 WHERE deployment_id = $1 AND device_id = $2`, [deploymentId, deviceId, e.message], ); } } static async cancel(deploymentId: number) { await query( `UPDATE fleet_deployments SET status = 'cancelled', completed_at = NOW() WHERE id = $1`, [deploymentId], ); await query( `UPDATE fleet_deployment_status SET status = 'cancelled', completed_at = NOW() WHERE deployment_id = $1 AND status IN ('pending','running')`, [deploymentId], ); return { success: true }; } /** * 롤백: 해당 배포의 이전 버전으로 복원 */ static async rollback(deploymentId: number, issuedBy?: string) { const deploy = await this.get(deploymentId); if (!deploy) throw new Error("배포 없음"); if (!["completed", "failed", "paused"].includes(deploy.status)) { throw new Error(`현재 상태(${deploy.status})에서 롤백 불가`); } // 이전 completed 배포 찾기 (같은 target) const previous = await queryOne( `SELECT d.*, r.backend_image, r.frontend_image, r.agent_image, r.version as release_version FROM fleet_deployments d LEFT JOIN fleet_releases r ON d.release_id = r.id WHERE d.id < $1 AND d.target_type = $2 AND d.target_value IS NOT DISTINCT FROM $3 AND d.status = 'completed' ORDER BY d.id DESC LIMIT 1`, [deploymentId, deploy.target_type, deploy.target_value], ); if (!previous) throw new Error("롤백할 이전 배포 없음"); const rollbackDeploy = await this.create({ release_id: previous.release_id, target_type: deploy.target_type, target_value: deploy.target_value, rollout_strategy: "immediate", rollout_percentage: 100, description: `롤백: #${deploymentId} → #${previous.id} (${previous.release_version})`, created_by: issuedBy, } as any); await this.start(rollbackDeploy.id, issuedBy); await query( `UPDATE fleet_deployments SET status = 'rolled_back' WHERE id = $1`, [deploymentId], ); return { deploymentId: rollbackDeploy.id, originalId: deploymentId, previousReleaseId: previous.release_id }; } }