diff --git a/backend-node/package-lock.json b/backend-node/package-lock.json index 096d549f..14ca59a4 100644 --- a/backend-node/package-lock.json +++ b/backend-node/package-lock.json @@ -16,8 +16,10 @@ "bcryptjs": "^2.4.3", "bwip-js": "^4.8.0", "cheerio": "^1.2.0", + "chokidar": "^3.6.0", "compression": "^1.7.4", "cors": "^2.8.5", + "csv-parse": "^5.6.0", "dockerode": "^4.0.10", "docx": "^9.5.1", "dotenv": "^16.3.1", @@ -48,7 +50,8 @@ "redis": "^4.6.10", "socket.io": "^4.8.3", "uuid": "^13.0.0", - "winston": "^3.11.0" + "winston": "^3.11.0", + "xlsx": "^0.18.5" }, "devDependencies": { "@types/bcryptjs": "^2.4.6", @@ -4196,6 +4199,15 @@ "node": ">=0.4.0" } }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/aedes": { "version": "0.51.3", "resolved": "https://registry.npmjs.org/aedes/-/aedes-0.51.3.tgz", @@ -4451,7 +4463,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "license": "ISC", "dependencies": { "normalize-path": "^3.0.0", @@ -4745,7 +4756,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5138,6 +5148,19 @@ ], "license": "CC-BY-4.0" }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "license": "Apache-2.0", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -5242,7 +5265,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", - "dev": true, "license": "MIT", "dependencies": { "anymatch": "~3.1.2", @@ -5267,7 +5289,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -5348,6 +5369,15 @@ "node": ">= 0.12.0" } }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/collect-v8-coverage": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", @@ -5605,6 +5635,18 @@ "node": ">=10.0.0" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/create-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", @@ -5677,6 +5719,12 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/csv-parse": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -7068,6 +7116,15 @@ "node": ">= 0.6" } }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", @@ -7094,7 +7151,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -8057,7 +8113,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "license": "MIT", "dependencies": { "binary-extensions": "^2.0.0" @@ -10030,7 +10085,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -11155,7 +11209,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "license": "MIT", "dependencies": { "picomatch": "^2.2.1" @@ -11890,6 +11943,18 @@ "node": ">= 0.6" } }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "license": "Apache-2.0", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/ssh2": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", @@ -12918,6 +12983,24 @@ "node": ">= 6" } }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -13065,6 +13148,27 @@ "resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz", "integrity": "sha512-GojqklwG8gpzOVEVki5KudKNoq7MbbjYZCbyWzEz7tyPA7eleiE0+ePwOWQQRb5fm86rD3S8Tc0tSFf3AOv50w==" }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "license": "Apache-2.0", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/xml": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", diff --git a/backend-node/package.json b/backend-node/package.json index c9a0d026..337f43e3 100644 --- a/backend-node/package.json +++ b/backend-node/package.json @@ -30,8 +30,10 @@ "bcryptjs": "^2.4.3", "bwip-js": "^4.8.0", "cheerio": "^1.2.0", + "chokidar": "^3.6.0", "compression": "^1.7.4", "cors": "^2.8.5", + "csv-parse": "^5.6.0", "dockerode": "^4.0.10", "docx": "^9.5.1", "dotenv": "^16.3.1", @@ -62,7 +64,8 @@ "redis": "^4.6.10", "socket.io": "^4.8.3", "uuid": "^13.0.0", - "winston": "^3.11.0" + "winston": "^3.11.0", + "xlsx": "^0.18.5" }, "devDependencies": { "@types/bcryptjs": "^2.4.6", diff --git a/backend-node/src/app.ts b/backend-node/src/app.ts index 4dd9a790..a75a6c9a 100644 --- a/backend-node/src/app.ts +++ b/backend-node/src/app.ts @@ -105,6 +105,7 @@ import externalDbConnectionRoutes from "./routes/externalDbConnectionRoutes"; import externalRestApiConnectionRoutes from "./routes/externalRestApiConnectionRoutes"; import centralForwarderRoutes from "./routes/centralForwarderRoutes"; import equipmentStateRoutes from "./routes/equipmentStateRoutes"; +import fileReaderRoutes from "./routes/fileReaderRoutes"; import automationDashboardRoutes from "./routes/automationDashboardRoutes"; import multiConnectionRoutes from "./routes/multiConnectionRoutes"; import screenFileRoutes from "./routes/screenFileRoutes"; @@ -348,6 +349,7 @@ app.use("/api/external-db-connections", externalDbConnectionRoutes); app.use("/api/external-rest-api-connections", externalRestApiConnectionRoutes); app.use("/api/central-forwarder", centralForwarderRoutes); app.use("/api/equipment-state", equipmentStateRoutes); +app.use("/api/file-reader", fileReaderRoutes); app.use("/api/automation-dashboard", automationDashboardRoutes); app.use("/api/multi-connection", multiConnectionRoutes); app.use("/api/screen-files", screenFileRoutes); @@ -522,6 +524,7 @@ async function initializeServices() { runDataTargetMigration, runEdgeDeviceIdentifierMigration, runFleetEdgeRawDeviceEndpointMigration, + runFileReaderTablesMigration, } = await import("./database/runMigration"); await runDashboardMigration(); @@ -542,6 +545,7 @@ async function initializeServices() { await runDataTargetMigration(); await runEdgeDeviceIdentifierMigration(); await runFleetEdgeRawDeviceEndpointMigration(); + await runFileReaderTablesMigration(); // 기본 데이터 소스 연결 시드 (IDC 엣지 관련 연결) const { seedDefaultDataSources } = await import( @@ -605,6 +609,12 @@ async function initializeServices() { ); startRetryWorker(); logger.info(`🔁 target DB retry worker 가동`); + + // File Reader 스케줄러 (watch 모드 cron 자동 등록) + const { startFileReaderScheduler } = await import( + "./services/fileReader/fileReaderScheduler" + ); + await startFileReaderScheduler(); } catch (error) { logger.error(`❌ 배치 스케줄러 초기화 실패:`, error); } diff --git a/backend-node/src/database/runMigration.ts b/backend-node/src/database/runMigration.ts index 2ef999a6..48e7b7b0 100644 --- a/backend-node/src/database/runMigration.ts +++ b/backend-node/src/database/runMigration.ts @@ -399,6 +399,21 @@ export async function runFleetEdgeRawDeviceEndpointMigration() { } } +export async function runFileReaderTablesMigration() { + try { + const sqlFilePath = path.join( + __dirname, + "../../db/migrations/313_create_file_reader_tables.sql" + ); + if (!fs.existsSync(sqlFilePath)) return; + const sqlContent = fs.readFileSync(sqlFilePath, "utf8"); + await PostgreSQLService.query(sqlContent); + console.log("✅ 파일 리더 테이블 (file_reader_configs/mappings/history) 생성 완료"); + } catch (error) { + console.error("❌ 파일 리더 테이블 마이그레이션 실패:", error); + } +} + export async function runOpenClawMigration() { try { console.log("🔄 OpenClaw AI 에이전트 마이그레이션 시작..."); diff --git a/backend-node/src/routes/fileReaderRoutes.ts b/backend-node/src/routes/fileReaderRoutes.ts new file mode 100644 index 00000000..dfd10f15 --- /dev/null +++ b/backend-node/src/routes/fileReaderRoutes.ts @@ -0,0 +1,323 @@ +/** + * File Reader REST 라우트 + * + * GET /api/file-reader/configs 목록 + * POST /api/file-reader/configs 생성 + * GET /api/file-reader/configs/:id 상세 + 매핑 + * PUT /api/file-reader/configs/:id 수정 + * DELETE /api/file-reader/configs/:id 삭제 + * + * POST /api/file-reader/configs/:id/preview 샘플 파일 미리보기 (multipart upload) + * POST /api/file-reader/configs/:id/run-upload 업로드 1회 적재 (multipart upload) + * POST /api/file-reader/configs/:id/run-watch watch 모드 즉시 1회 스캔 + * + * GET /api/file-reader/configs/:id/history 이력 + * GET /api/file-reader/allowed-roots 허용 마운트 루트 목록 + */ +import { Router, Response } from "express"; +import multer from "multer"; +import path from "path"; +import fs from "fs"; +import os from "os"; +import { authenticateToken } from "../middleware/authMiddleware"; +import { AuthenticatedRequest } from "../types/auth"; +import { query as localQuery } from "../database/db"; +import { logger } from "../utils/logger"; +import { parseFile } from "../services/fileReader/parsers"; +import { + getAllowedRoots, + isPathAllowed, + processFile, + scanAndProcess, + FileReaderConfig, +} from "../services/fileReader/fileReaderService"; + +const router = Router(); + +const upload = multer({ + storage: multer.diskStorage({ + destination: (_req, _file, cb) => { + const tmp = path.join(os.tmpdir(), "file-reader-uploads"); + if (!fs.existsSync(tmp)) fs.mkdirSync(tmp, { recursive: true }); + cb(null, tmp); + }, + filename: (_req, file, cb) => { + const safe = file.originalname.replace(/[^\w.\-가-힣]/g, "_"); + cb(null, `${Date.now()}_${safe}`); + }, + }), + limits: { fileSize: 200 * 1024 * 1024 }, // 200MB +}); + +// ─── 허용 루트 ───────────────────────────────────── +router.get( + "/allowed-roots", + authenticateToken, + async (_req: AuthenticatedRequest, res: Response) => { + res.json({ success: true, roots: getAllowedRoots() }); + } +); + +// ─── 목록 ────────────────────────────────────────── +router.get( + "/configs", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const companyCode = (req.query.company_code as string) || req.user?.companyCode || "*"; + const rows = await localQuery( + `SELECT * FROM file_reader_configs + WHERE company_code = $1 OR company_code = '*' OR $1 = '*' + ORDER BY id DESC`, + [companyCode] + ); + res.json({ success: true, items: rows }); + } +); + +// ─── 상세 ────────────────────────────────────────── +router.get( + "/configs/:id", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const id = Number(req.params.id); + const cfg = await localQuery( + `SELECT * FROM file_reader_configs WHERE id = $1`, + [id] + ); + if (cfg.length === 0) return res.status(404).json({ success: false, error: "없음" }); + const mappings = await localQuery( + `SELECT * FROM file_reader_mappings WHERE config_id = $1 ORDER BY mapping_order, id`, + [id] + ); + res.json({ success: true, config: cfg[0], mappings }); + } +); + +// ─── 생성 ────────────────────────────────────────── +router.post( + "/configs", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const b = req.body || {}; + if (!b.name || !b.file_type) { + return res.status(400).json({ success: false, error: "name, file_type 필수" }); + } + if (b.source_mode === "watch" && b.host_path && !isPathAllowed(b.host_path)) { + return res.status(400).json({ + success: false, + error: `허용되지 않은 호스트 경로: ${b.host_path}`, + allowed_roots: getAllowedRoots(), + }); + } + const cc = b.company_code || req.user?.companyCode || "*"; + const ins = await localQuery<{ id: number }>( + `INSERT INTO file_reader_configs + (name, company_code, file_type, source_mode, host_path, file_pattern, + processed_action, archive_subdir, has_header, delimiter, encoding, sheet_name, skip_rows, + target_db_id, target_schema, target_table, save_mode, conflict_keys, + cron_schedule, is_active, description, created_by) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22) + RETURNING id`, + [ + b.name, cc, b.file_type, b.source_mode || "upload", + b.host_path || null, b.file_pattern || "*.csv", + b.processed_action || "mark", b.archive_subdir || "_processed", + b.has_header !== false, b.delimiter || ",", b.encoding || "utf-8", + b.sheet_name || null, b.skip_rows || 0, + b.target_db_id || null, b.target_schema || "public", b.target_table || null, + b.save_mode || "INSERT", b.conflict_keys || null, + b.cron_schedule || null, b.is_active || "Y", + b.description || null, req.user?.userId || "system", + ] + ); + const id = ins[0].id; + + if (Array.isArray(b.mappings)) { + for (let i = 0; i < b.mappings.length; i++) { + const m = b.mappings[i]; + if (!m.from_column || !m.to_column) continue; + await localQuery( + `INSERT INTO file_reader_mappings + (config_id, from_column, from_column_type, to_column, to_data_type, + default_value, transform_expr, is_required, mapping_order) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, + [ + id, m.from_column, m.from_column_type || "header", + m.to_column, m.to_data_type || "TEXT", + m.default_value || null, m.transform_expr || null, + !!m.is_required, m.mapping_order ?? i, + ] + ); + } + } + res.json({ success: true, id }); + } +); + +// ─── 수정 ────────────────────────────────────────── +router.put( + "/configs/:id", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const id = Number(req.params.id); + const b = req.body || {}; + if (b.source_mode === "watch" && b.host_path && !isPathAllowed(b.host_path)) { + return res.status(400).json({ + success: false, + error: `허용되지 않은 호스트 경로: ${b.host_path}`, + allowed_roots: getAllowedRoots(), + }); + } + await localQuery( + `UPDATE file_reader_configs SET + name=$1, file_type=$2, source_mode=$3, host_path=$4, file_pattern=$5, + processed_action=$6, archive_subdir=$7, has_header=$8, delimiter=$9, encoding=$10, + sheet_name=$11, skip_rows=$12, target_db_id=$13, target_schema=$14, target_table=$15, + save_mode=$16, conflict_keys=$17, cron_schedule=$18, is_active=$19, + description=$20, updated_by=$21, updated_date=NOW() + WHERE id=$22`, + [ + b.name, b.file_type, b.source_mode || "upload", + b.host_path || null, b.file_pattern || "*.csv", + b.processed_action || "mark", b.archive_subdir || "_processed", + b.has_header !== false, b.delimiter || ",", b.encoding || "utf-8", + b.sheet_name || null, b.skip_rows || 0, + b.target_db_id || null, b.target_schema || "public", b.target_table || null, + b.save_mode || "INSERT", b.conflict_keys || null, + b.cron_schedule || null, b.is_active || "Y", + b.description || null, req.user?.userId || "system", id, + ] + ); + + if (Array.isArray(b.mappings)) { + await localQuery(`DELETE FROM file_reader_mappings WHERE config_id = $1`, [id]); + for (let i = 0; i < b.mappings.length; i++) { + const m = b.mappings[i]; + if (!m.from_column || !m.to_column) continue; + await localQuery( + `INSERT INTO file_reader_mappings + (config_id, from_column, from_column_type, to_column, to_data_type, + default_value, transform_expr, is_required, mapping_order) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, + [ + id, m.from_column, m.from_column_type || "header", + m.to_column, m.to_data_type || "TEXT", + m.default_value || null, m.transform_expr || null, + !!m.is_required, m.mapping_order ?? i, + ] + ); + } + } + res.json({ success: true }); + } +); + +// ─── 삭제 ────────────────────────────────────────── +router.delete( + "/configs/:id", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const id = Number(req.params.id); + await localQuery(`DELETE FROM file_reader_configs WHERE id = $1`, [id]); + res.json({ success: true }); + } +); + +// ─── 샘플 미리보기 (multipart) ───────────────────── +router.post( + "/configs/:id/preview", + authenticateToken, + upload.single("file"), + async (req: AuthenticatedRequest, res: Response) => { + if (!req.file) return res.status(400).json({ success: false, error: "파일 없음" }); + const id = Number(req.params.id); + const cfgs = await localQuery( + `SELECT * FROM file_reader_configs WHERE id = $1`, + [id] + ); + if (cfgs.length === 0) { + fs.unlinkSync(req.file.path); + return res.status(404).json({ success: false, error: "config 없음" }); + } + const c = cfgs[0]; + try { + const parsed = parseFile(req.file.path, { + fileType: c.file_type, + hasHeader: c.has_header, + delimiter: c.delimiter || undefined, + encoding: c.encoding || undefined, + sheetName: c.sheet_name, + skipRows: c.skip_rows || 0, + }); + res.json({ + success: true, + headers: parsed.headers, + totalRows: parsed.rows.length, + sampleRows: parsed.rows.slice(0, 20), + }); + } catch (e) { + res.status(400).json({ success: false, error: (e as Error).message }); + } finally { + fs.unlink(req.file.path, () => undefined); + } + } +); + +// ─── 업로드 → 1회 적재 ──────────────────────────── +router.post( + "/configs/:id/run-upload", + authenticateToken, + upload.single("file"), + async (req: AuthenticatedRequest, res: Response) => { + if (!req.file) return res.status(400).json({ success: false, error: "파일 없음" }); + const id = Number(req.params.id); + const cfgs = await localQuery( + `SELECT * FROM file_reader_configs WHERE id = $1`, + [id] + ); + if (cfgs.length === 0) { + fs.unlinkSync(req.file.path); + return res.status(404).json({ success: false, error: "config 없음" }); + } + const result = await processFile(cfgs[0], req.file.path); + fs.unlink(req.file.path, () => undefined); + res.json({ success: result.status !== "failure", ...result }); + } +); + +// ─── watch 모드: 즉시 1회 스캔 ──────────────────── +router.post( + "/configs/:id/run-watch", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const id = Number(req.params.id); + const cfgs = await localQuery( + `SELECT * FROM file_reader_configs WHERE id = $1`, + [id] + ); + if (cfgs.length === 0) return res.status(404).json({ success: false, error: "config 없음" }); + try { + const r = await scanAndProcess(cfgs[0]); + res.json({ success: true, ...r }); + } catch (e) { + res.status(400).json({ success: false, error: (e as Error).message }); + } + } +); + +// ─── 이력 ────────────────────────────────────────── +router.get( + "/configs/:id/history", + authenticateToken, + async (req: AuthenticatedRequest, res: Response) => { + const id = Number(req.params.id); + const limit = Math.min(Number(req.query.limit) || 50, 500); + const rows = await localQuery( + `SELECT * FROM file_reader_history WHERE config_id = $1 ORDER BY started_at DESC LIMIT $2`, + [id, limit] + ); + res.json({ success: true, items: rows }); + } +); + +export default router; diff --git a/backend-node/src/services/fileReader/fileReaderScheduler.ts b/backend-node/src/services/fileReader/fileReaderScheduler.ts new file mode 100644 index 00000000..095480cc --- /dev/null +++ b/backend-node/src/services/fileReader/fileReaderScheduler.ts @@ -0,0 +1,72 @@ +/** + * File Reader watch 모드 스케줄러 + * cron_schedule 이 설정된 활성 watch 설정마다 cron 등록 → 주기 scanAndProcess + */ +import cron, { ScheduledTask } from "node-cron"; +import { logger } from "../../utils/logger"; +import { query } from "../../database/db"; +import { scanAndProcess, FileReaderConfig } from "./fileReaderService"; + +const tasks = new Map(); + +export async function startFileReaderScheduler(): Promise { + if (process.env.FILE_READER_SCHEDULER_DISABLED === "true") { + logger.info("[FileReader] 스케줄러 비활성 (FILE_READER_SCHEDULER_DISABLED=true)"); + return; + } + await reloadAll(); + // 1분마다 설정 변화 감지 + 재등록 + setInterval(() => { + reloadAll().catch(e => logger.warn(`[FileReader] reload 실패: ${e.message}`)); + }, 60_000); + logger.info("[FileReader] 스케줄러 시작 (1분마다 설정 reload)"); +} + +export function stopFileReaderScheduler(): void { + for (const [, t] of tasks) t.stop(); + tasks.clear(); +} + +async function reloadAll(): Promise { + const rows = await query( + `SELECT * FROM file_reader_configs + WHERE source_mode = 'watch' + AND is_active = 'Y' + AND cron_schedule IS NOT NULL + AND host_path IS NOT NULL` + ); + + const currentIds = new Set(rows.map(r => r.id)); + + // 사라진 설정 정리 + for (const [id, task] of tasks) { + if (!currentIds.has(id)) { + task.stop(); + tasks.delete(id); + logger.info(`[FileReader] 스케줄 제거 config=${id}`); + } + } + + for (const cfg of rows) { + const existing = tasks.get(cfg.id); + if (existing) continue; // 변경 감지는 단순화 — 삭제 후 재추가는 PUT 라우트가 책임 + if (!cfg.cron_schedule || !cron.validate(cfg.cron_schedule)) { + logger.warn(`[FileReader] 잘못된 cron 표현식 config=${cfg.id}: ${cfg.cron_schedule}`); + continue; + } + const task = cron.schedule(cfg.cron_schedule, async () => { + try { + const r = await scanAndProcess(cfg); + if (r.scanned > 0 || r.processed > 0 || r.failed > 0) { + logger.info( + `[FileReader] watch tick config=${cfg.id} scanned=${r.scanned} processed=${r.processed} skipped=${r.skipped} failed=${r.failed}` + ); + } + } catch (e) { + logger.warn(`[FileReader] watch tick 실패 config=${cfg.id}: ${(e as Error).message}`); + } + }); + tasks.set(cfg.id, task); + logger.info(`[FileReader] 스케줄 등록 config=${cfg.id} cron='${cfg.cron_schedule}' path=${cfg.host_path}`); + } +} diff --git a/backend-node/src/services/fileReader/fileReaderService.ts b/backend-node/src/services/fileReader/fileReaderService.ts new file mode 100644 index 00000000..f0331bf8 --- /dev/null +++ b/backend-node/src/services/fileReader/fileReaderService.ts @@ -0,0 +1,337 @@ +/** + * File Reader 서비스 + * + * - 호스트 경로 검증 (FILE_READER_ALLOWED_ROOTS 환경변수 기반) + * - 파일 파싱 → 매핑 적용 → 타겟 DB INSERT + * - 처리 이력 기록 + */ +import fs from "fs"; +import path from "path"; +import { query as localQuery } from "../../database/db"; +import { logger } from "../../utils/logger"; +import { executeExternalQuery } from "../externalDbHelper"; +import { parseFile, castValue, applyTransform } from "./parsers"; + +export interface FileReaderConfig { + id: number; + name: string; + company_code: string; + file_type: "csv" | "tsv" | "txt" | "xlsx" | "xls"; + source_mode: "upload" | "watch"; + host_path: string | null; + file_pattern: string | null; + processed_action: "archive" | "delete" | "mark"; + archive_subdir: string | null; + has_header: boolean; + delimiter: string | null; + encoding: string | null; + sheet_name: string | null; + skip_rows: number | null; + target_db_id: number | null; + target_schema: string | null; + target_table: string | null; + save_mode: "INSERT" | "UPSERT" | "REPLACE"; + conflict_keys: string | null; + cron_schedule: string | null; + is_active: string; +} + +export interface FileReaderMapping { + id: number; + config_id: number; + from_column: string; + from_column_type: "header" | "index"; + to_column: string; + to_data_type: string; + default_value: string | null; + transform_expr: string | null; + is_required: boolean; + mapping_order: number; +} + +/** + * 허용 마운트 루트 확인 + */ +export function getAllowedRoots(): string[] { + const raw = process.env.FILE_READER_ALLOWED_ROOTS || "/home/wace/file-imports,/mnt"; + return raw + .split(",") + .map(s => s.trim()) + .filter(Boolean); +} + +export function isPathAllowed(absPath: string): boolean { + if (!path.isAbsolute(absPath)) return false; + const normalized = path.resolve(absPath); + return getAllowedRoots().some(root => { + const r = path.resolve(root); + return normalized === r || normalized.startsWith(r + path.sep); + }); +} + +/** + * 단일 파일 처리: 파싱 → 매핑 → INSERT → 이력 기록 + */ +export async function processFile( + config: FileReaderConfig, + absFilePath: string +): Promise<{ + status: "success" | "failure" | "partial" | "skipped"; + rowsTotal: number; + rowsInserted: number; + rowsFailed: number; + errorMessage: string | null; +}> { + let historyId: number | null = null; + try { + if (!fs.existsSync(absFilePath)) { + throw new Error(`파일을 찾을 수 없습니다: ${absFilePath}`); + } + const stat = fs.statSync(absFilePath); + + // 중복 처리 방지 (success/skipped만 차단) + const dup = await localQuery<{ id: number }>( + `SELECT id FROM file_reader_history + WHERE config_id = $1 AND file_path = $2 AND status IN ('success','skipped')`, + [config.id, absFilePath] + ); + if (dup.length > 0) { + return { + status: "skipped", + rowsTotal: 0, + rowsInserted: 0, + rowsFailed: 0, + errorMessage: "이미 처리된 파일", + }; + } + + // 이력 시작 + const ins = await localQuery<{ id: number }>( + `INSERT INTO file_reader_history + (config_id, file_path, file_size, file_mtime, started_at, status) + VALUES ($1, $2, $3, $4, NOW(), 'success') + RETURNING id`, + [config.id, absFilePath, stat.size, stat.mtime] + ); + historyId = ins[0].id; + + // 파싱 + const parsed = parseFile(absFilePath, { + fileType: config.file_type, + hasHeader: config.has_header, + delimiter: config.delimiter || undefined, + encoding: config.encoding || undefined, + sheetName: config.sheet_name, + skipRows: config.skip_rows || 0, + }); + + if (!config.target_db_id || !config.target_table) { + throw new Error("target_db_id 또는 target_table 미설정"); + } + + // 매핑 + const mappings = await localQuery( + `SELECT * FROM file_reader_mappings WHERE config_id = $1 ORDER BY mapping_order, id`, + [config.id] + ); + if (mappings.length === 0) throw new Error("컬럼 매핑 없음"); + + // 컬럼 식별 + const resolveFromValue = ( + row: Record, + m: FileReaderMapping + ): unknown => { + const key = m.from_column; + if (m.from_column_type === "index") { + // index 모드: row가 header-keyed면 keys 배열 i번째 + const keys = Object.keys(row); + const idx = Number(key); + return Number.isFinite(idx) && idx < keys.length ? row[keys[idx]] : undefined; + } + return row[key]; + }; + + // 적재 + const targetCols = mappings.map(m => `"${m.to_column}"`); + const fqTable = `"${config.target_schema || "public"}"."${config.target_table}"`; + + let inserted = 0; + let failed = 0; + const batchSize = 100; + + for (let i = 0; i < parsed.rows.length; i += batchSize) { + const batch = parsed.rows.slice(i, i + batchSize); + const values: unknown[] = []; + const placeholders: string[] = []; + + let validCount = 0; + for (const row of batch) { + const mapped: unknown[] = []; + let rowOk = true; + for (const m of mappings) { + let v = resolveFromValue(row, m); + if ((v == null || v === "") && m.default_value != null) v = m.default_value; + if ((v == null || v === "") && m.is_required) { + rowOk = false; + break; + } + v = applyTransform(v, m.transform_expr); + v = castValue(v, m.to_data_type || "TEXT"); + mapped.push(v); + } + if (!rowOk) { + failed++; + continue; + } + const base = values.length; + placeholders.push( + "(" + mapped.map((_, k) => `$${base + k + 1}`).join(",") + ")" + ); + values.push(...mapped); + validCount++; + } + + if (validCount === 0) continue; + + let sql = `INSERT INTO ${fqTable} (${targetCols.join(",")}) VALUES ${placeholders.join(",")}`; + if (config.save_mode === "UPSERT" && config.conflict_keys) { + const keys = config.conflict_keys + .split(",") + .map(s => `"${s.trim()}"`) + .filter(Boolean); + const upd = targetCols + .filter(c => !keys.includes(c)) + .map(c => `${c} = EXCLUDED.${c}`) + .join(", "); + sql += ` ON CONFLICT (${keys.join(",")}) DO UPDATE SET ${upd}`; + } else if (config.save_mode === "REPLACE" && i === 0) { + await executeExternalQuery(config.target_db_id, `DELETE FROM ${fqTable}`, []); + } + + try { + await executeExternalQuery(config.target_db_id, sql, values); + inserted += validCount; + } catch (err) { + failed += validCount; + logger.warn( + `[FileReader] 배치 INSERT 실패 config=${config.id} file=${absFilePath}: ${(err as Error).message}` + ); + } + } + + const finalStatus: "success" | "partial" | "failure" = + failed === 0 ? "success" : inserted === 0 ? "failure" : "partial"; + + await localQuery( + `UPDATE file_reader_history + SET finished_at=NOW(), status=$1, rows_total=$2, rows_inserted=$3, rows_failed=$4 + WHERE id=$5`, + [finalStatus, parsed.rows.length, inserted, failed, historyId] + ); + + await localQuery( + `UPDATE file_reader_configs + SET last_run_at=NOW(), last_run_result=$1, last_run_message=$2, last_processed_count=$3, updated_date=NOW() + WHERE id=$4`, + [finalStatus, `processed=${inserted}/${parsed.rows.length} failed=${failed}`, inserted, config.id] + ); + + // 처리 완료 파일 정리 (watch 모드) + if (config.source_mode === "watch" && finalStatus !== "failure") { + try { + if (config.processed_action === "delete") { + fs.unlinkSync(absFilePath); + } else if (config.processed_action === "archive") { + const archDir = path.join( + path.dirname(absFilePath), + config.archive_subdir || "_processed" + ); + if (!fs.existsSync(archDir)) fs.mkdirSync(archDir, { recursive: true }); + fs.renameSync(absFilePath, path.join(archDir, path.basename(absFilePath))); + } + } catch (e) { + logger.warn(`[FileReader] 후처리 실패 ${absFilePath}: ${(e as Error).message}`); + } + } + + return { + status: finalStatus, + rowsTotal: parsed.rows.length, + rowsInserted: inserted, + rowsFailed: failed, + errorMessage: null, + }; + } catch (err) { + const msg = (err as Error).message; + logger.error(`[FileReader] 파일 처리 실패 config=${config.id} ${absFilePath}: ${msg}`); + if (historyId) { + await localQuery( + `UPDATE file_reader_history SET finished_at=NOW(), status='failure', error_message=$1 WHERE id=$2`, + [msg, historyId] + ).catch(() => undefined); + } + await localQuery( + `UPDATE file_reader_configs + SET last_run_at=NOW(), last_run_result='failure', last_run_message=$1, updated_date=NOW() + WHERE id=$2`, + [msg, config.id] + ).catch(() => undefined); + return { + status: "failure", + rowsTotal: 0, + rowsInserted: 0, + rowsFailed: 0, + errorMessage: msg, + }; + } +} + +/** + * watch 모드: host_path 디렉토리에서 file_pattern 매칭 파일 모두 처리 + */ +export async function scanAndProcess(config: FileReaderConfig): Promise<{ + scanned: number; + processed: number; + skipped: number; + failed: number; +}> { + if (config.source_mode !== "watch" || !config.host_path) { + return { scanned: 0, processed: 0, skipped: 0, failed: 0 }; + } + if (!isPathAllowed(config.host_path)) { + throw new Error( + `허용되지 않은 경로: ${config.host_path}. FILE_READER_ALLOWED_ROOTS 확인 필요` + ); + } + if (!fs.existsSync(config.host_path) || !fs.statSync(config.host_path).isDirectory()) { + throw new Error(`디렉토리 없음: ${config.host_path}`); + } + + const pattern = config.file_pattern || "*"; + const regex = globToRegex(pattern); + const archive = config.archive_subdir || "_processed"; + + const files = fs + .readdirSync(config.host_path, { withFileTypes: true }) + .filter(d => d.isFile() && d.name !== archive && regex.test(d.name)) + .map(d => path.join(config.host_path!, d.name)); + + let processed = 0; + let skipped = 0; + let failed = 0; + for (const f of files) { + const r = await processFile(config, f); + if (r.status === "skipped") skipped++; + else if (r.status === "failure") failed++; + else processed++; + } + return { scanned: files.length, processed, skipped, failed }; +} + +function globToRegex(glob: string): RegExp { + const escaped = glob + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*") + .replace(/\?/g, "."); + return new RegExp(`^${escaped}$`, "i"); +} diff --git a/backend-node/src/services/fileReader/parsers.ts b/backend-node/src/services/fileReader/parsers.ts new file mode 100644 index 00000000..9487aa5f --- /dev/null +++ b/backend-node/src/services/fileReader/parsers.ts @@ -0,0 +1,206 @@ +/** + * File Reader 파서 모듈 + * CSV / TSV / TXT / XLSX → array of row records 로 변환 + */ +import fs from "fs"; +import { parse as csvParse } from "csv-parse/sync"; +import * as XLSX from "xlsx"; + +export interface ParseOptions { + fileType: "csv" | "tsv" | "txt" | "xlsx" | "xls"; + hasHeader: boolean; + delimiter?: string; + encoding?: string; + sheetName?: string | null; + skipRows?: number; +} + +export interface ParseResult { + headers: string[]; + rows: Record[]; +} + +export function parseFile(absPath: string, opts: ParseOptions): ParseResult { + switch (opts.fileType) { + case "csv": + case "tsv": + case "txt": + return parseDelimited(absPath, opts); + case "xlsx": + case "xls": + return parseExcel(absPath, opts); + default: + throw new Error(`지원하지 않는 파일 유형: ${opts.fileType}`); + } +} + +function parseDelimited(absPath: string, opts: ParseOptions): ParseResult { + const buf = fs.readFileSync(absPath); + const encoding = (opts.encoding || "utf-8").toLowerCase(); + let text: string; + if (encoding === "utf-8" || encoding === "utf8") { + text = buf.toString("utf-8"); + if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); // BOM 제거 + } else { + text = buf.toString(encoding as BufferEncoding); + } + + const delimiter = + opts.delimiter || + (opts.fileType === "tsv" ? "\t" : opts.fileType === "txt" ? "\t" : ","); + + const skip = opts.skipRows || 0; + const lines = text.split(/\r?\n/); + const effective = lines.slice(skip).join("\n"); + + const records = csvParse(effective, { + delimiter, + columns: opts.hasHeader, + skip_empty_lines: true, + relax_quotes: true, + relax_column_count: true, + trim: true, + }) as Record[] | string[][]; + + let headers: string[]; + let rows: Record[]; + + if (opts.hasHeader) { + const dataRows = records as Record[]; + headers = dataRows.length ? Object.keys(dataRows[0]) : []; + rows = dataRows; + } else { + const arrRows = records as string[][]; + headers = arrRows.length ? arrRows[0].map((_, i) => String(i)) : []; + rows = arrRows.map(r => { + const obj: Record = {}; + r.forEach((v, i) => { + obj[String(i)] = v; + }); + return obj; + }); + } + + return { headers, rows }; +} + +function parseExcel(absPath: string, opts: ParseOptions): ParseResult { + const wb = XLSX.readFile(absPath, { cellDates: true }); + const sheetName = + opts.sheetName && wb.SheetNames.includes(opts.sheetName) + ? opts.sheetName + : wb.SheetNames[0]; + if (!sheetName) throw new Error("시트가 없습니다"); + + const sheet = wb.Sheets[sheetName]; + const skip = opts.skipRows || 0; + + const aoa = XLSX.utils.sheet_to_json(sheet, { + header: 1, + raw: true, + blankrows: false, + }); + const sliced = aoa.slice(skip); + if (sliced.length === 0) return { headers: [], rows: [] }; + + let headers: string[]; + let dataRows: unknown[][]; + if (opts.hasHeader) { + headers = (sliced[0] as unknown[]).map((h, i) => + h == null || h === "" ? `col${i}` : String(h).trim() + ); + dataRows = sliced.slice(1) as unknown[][]; + } else { + const first = sliced[0] as unknown[]; + headers = first.map((_, i) => String(i)); + dataRows = sliced as unknown[][]; + } + + const rows = dataRows.map(arr => { + const obj: Record = {}; + headers.forEach((h, i) => { + const v = arr[i]; + if (v == null || v === "") obj[h] = null; + else if (v instanceof Date) obj[h] = v.toISOString(); + else if (typeof v === "number" || typeof v === "boolean") obj[h] = v; + else obj[h] = String(v); + }); + return obj; + }); + + return { headers, rows }; +} + +/** + * 값 타입 캐스팅 (매핑의 to_data_type에 따라) + */ +export function castValue(raw: unknown, dataType: string): unknown { + if (raw === null || raw === undefined || raw === "") return null; + const t = dataType.toUpperCase(); + try { + switch (t) { + case "INTEGER": + case "INT": + case "BIGINT": { + const n = Number(String(raw).replace(/,/g, "")); + return Number.isFinite(n) ? Math.trunc(n) : null; + } + case "NUMERIC": + case "DECIMAL": + case "FLOAT": + case "DOUBLE": + case "REAL": { + const n = Number(String(raw).replace(/,/g, "")); + return Number.isFinite(n) ? n : null; + } + case "BOOLEAN": + case "BOOL": { + const s = String(raw).toLowerCase().trim(); + if (["true", "1", "y", "yes", "t"].includes(s)) return true; + if (["false", "0", "n", "no", "f"].includes(s)) return false; + return null; + } + case "DATE": + case "TIMESTAMP": + case "TIMESTAMPTZ": + case "DATETIME": { + const d = raw instanceof Date ? raw : new Date(String(raw)); + return isNaN(d.getTime()) ? null : d.toISOString(); + } + case "JSON": + case "JSONB": { + return typeof raw === "string" ? raw : JSON.stringify(raw); + } + default: + return String(raw); + } + } catch { + return null; + } +} + +/** + * 매우 간단한 transform 적용 (TRIM / UPPER / LOWER / x*N 등) + */ +export function applyTransform(value: unknown, expr: string | null): unknown { + if (!expr) return value; + const e = expr.trim(); + if (value == null) return value; + if (e.toUpperCase() === "TRIM") return String(value).trim(); + if (e.toUpperCase() === "UPPER") return String(value).toUpperCase(); + if (e.toUpperCase() === "LOWER") return String(value).toLowerCase(); + // 단순 산술: "x*1000", "x+1", "x/100" + const m = e.match(/^x\s*([*+\-/])\s*(-?\d+(?:\.\d+)?)$/i); + if (m) { + const n = Number(value); + if (!Number.isFinite(n)) return null; + const k = Number(m[2]); + switch (m[1]) { + case "*": return n * k; + case "/": return k === 0 ? null : n / k; + case "+": return n + k; + case "-": return n - k; + } + } + return value; +} diff --git a/frontend/app/(main)/admin/automaticMng/exconList/page.tsx b/frontend/app/(main)/admin/automaticMng/exconList/page.tsx index e81e8a6e..b28aaa8f 100644 --- a/frontend/app/(main)/admin/automaticMng/exconList/page.tsx +++ b/frontend/app/(main)/admin/automaticMng/exconList/page.tsx @@ -26,6 +26,7 @@ import { import { ExternalDbConnectionModal } from "@/components/admin/ExternalDbConnectionModal"; import { SqlQueryModal } from "@/components/admin/SqlQueryModal"; import { RestApiConnectionList } from "@/components/admin/RestApiConnectionList"; +import { FileReaderConnectionList } from "@/components/admin/FileReaderConnectionList"; import { ScrollToTop } from "@/components/common/ScrollToTop"; import dynamic from "next/dynamic"; @@ -390,7 +391,7 @@ export default function DataSourcePage() { {/* 파일 리더 탭 */} - + diff --git a/frontend/components/admin/FileReaderConnectionList.tsx b/frontend/components/admin/FileReaderConnectionList.tsx new file mode 100644 index 00000000..5fa129c6 --- /dev/null +++ b/frontend/components/admin/FileReaderConnectionList.tsx @@ -0,0 +1,204 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { Plus, Pencil, Trash2, Play, FileText, RefreshCcw } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Badge } from "@/components/ui/badge"; +import { + AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent, + AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { useToast } from "@/hooks/use-toast"; +import { FileReaderAPI, FileReaderConfig } from "@/lib/api/fileReader"; +import { FileReaderConnectionModal } from "./FileReaderConnectionModal"; + +const RESULT_BADGE: Record = { + success: "bg-green-100 text-green-700", + partial: "bg-amber-100 text-amber-700", + failure: "bg-rose-100 text-rose-700", + skipped: "bg-gray-100 text-gray-600", +}; + +export const FileReaderConnectionList: React.FC = () => { + const { toast } = useToast(); + const [items, setItems] = useState([]); + const [search, setSearch] = useState(""); + const [loading, setLoading] = useState(false); + const [modalOpen, setModalOpen] = useState(false); + const [editingId, setEditingId] = useState(null); + const [confirmDel, setConfirmDel] = useState(null); + + const load = async () => { + setLoading(true); + try { + const list = await FileReaderAPI.list(); + setItems(list); + } catch (e) { + toast({ title: "조회 실패", description: (e as Error).message, variant: "destructive" }); + } finally { + setLoading(false); + } + }; + useEffect(() => { void load(); }, []); + + const handleRunUpload = async (cfg: FileReaderConfig) => { + const inp = document.createElement("input"); + inp.type = "file"; + inp.accept = ".csv,.tsv,.txt,.xlsx,.xls"; + inp.onchange = async () => { + if (!inp.files || inp.files.length === 0) return; + try { + const r = await FileReaderAPI.runUpload(cfg.id!, inp.files[0]); + toast({ + title: r.status === "failure" ? "적재 실패" : "적재 완료", + description: `총 ${r.rowsTotal}행 / 성공 ${r.rowsInserted} / 실패 ${r.rowsFailed}`, + variant: r.status === "failure" ? "destructive" : "default", + }); + void load(); + } catch (e) { + toast({ title: "실패", description: (e as Error).message, variant: "destructive" }); + } + }; + inp.click(); + }; + + const handleRunWatch = async (cfg: FileReaderConfig) => { + try { + const r = await FileReaderAPI.runWatch(cfg.id!); + toast({ + title: "스캔 완료", + description: `스캔 ${r.scanned} / 처리 ${r.processed} / 스킵 ${r.skipped} / 실패 ${r.failed}`, + }); + void load(); + } catch (e) { + toast({ title: "스캔 실패", description: (e as Error).message, variant: "destructive" }); + } + }; + + const filtered = items.filter(i => + !search || i.name.toLowerCase().includes(search.toLowerCase()) || + (i.target_table || "").toLowerCase().includes(search.toLowerCase()) + ); + + return ( +
+
+ setSearch(e.target.value)} + className="max-w-sm" + /> +
+ + +
+
+ + {filtered.length === 0 ? ( +
+ + 등록된 파일 리더가 없습니다. "새 파일 리더" 버튼으로 추가하세요. +
+ ) : ( +
+ {filtered.map(cfg => ( +
+
+
+
+ {cfg.name} +
+
+ {cfg.file_type.toUpperCase()} · {cfg.source_mode === "watch" ? "감시 폴더" : "업로드"} · + {" "}target: {cfg.target_schema || "public"}.{cfg.target_table || "(미지정)"} +
+
+ + {cfg.is_active === "Y" ? "활성" : "비활성"} + +
+ {cfg.source_mode === "watch" && cfg.host_path && ( +
+ 📁 {cfg.host_path}/{cfg.file_pattern || "*"} +
+ )} + {cfg.last_run_at && ( +
+ + {cfg.last_run_result || "-"} + + + {new Date(cfg.last_run_at).toLocaleString("ko-KR")} · 행 {cfg.last_processed_count || 0} + +
+ )} +
+ {cfg.source_mode === "watch" ? ( + + ) : ( + + )} + + +
+
+ ))} +
+ )} + + {modalOpen && ( + setModalOpen(false)} + onSaved={() => { setModalOpen(false); void load(); }} + /> + )} + + !o && setConfirmDel(null)}> + + + 파일 리더 삭제 + + "{confirmDel?.name}" 설정과 컬럼 매핑, 처리 이력을 모두 삭제합니다. 계속하시겠습니까? + + + + 취소 + { + if (!confirmDel) return; + try { + await FileReaderAPI.remove(confirmDel.id!); + toast({ title: "삭제 완료" }); + setConfirmDel(null); + void load(); + } catch (e) { + toast({ title: "삭제 실패", description: (e as Error).message, variant: "destructive" }); + } + }} + >삭제 + + + +
+ ); +}; + +export default FileReaderConnectionList; diff --git a/frontend/components/admin/FileReaderConnectionModal.tsx b/frontend/components/admin/FileReaderConnectionModal.tsx new file mode 100644 index 00000000..8327ae74 --- /dev/null +++ b/frontend/components/admin/FileReaderConnectionModal.tsx @@ -0,0 +1,440 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import { Switch } from "@/components/ui/switch"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Plus, Trash2, Upload, X } from "lucide-react"; +import { useToast } from "@/hooks/use-toast"; +import { + FileReaderAPI, FileReaderConfig, FileReaderMapping, FileType, SourceMode, SaveMode, +} from "@/lib/api/fileReader"; +import { ExternalDbConnectionAPI } from "@/lib/api/externalDbConnection"; + +interface Props { + open: boolean; + editingId: number | null; + onClose: () => void; + onSaved: () => void; +} + +const defaultConfig: Partial = { + name: "", + company_code: "*", + file_type: "csv", + source_mode: "upload", + has_header: true, + delimiter: ",", + encoding: "utf-8", + skip_rows: 0, + target_schema: "public", + save_mode: "INSERT", + is_active: "Y", + file_pattern: "*.csv", + processed_action: "mark", +}; + +export const FileReaderConnectionModal: React.FC = ({ open, editingId, onClose, onSaved }) => { + const { toast } = useToast(); + const [cfg, setCfg] = useState>(defaultConfig); + const [mappings, setMappings] = useState([]); + const [dbList, setDbList] = useState>([]); + const [allowedRoots, setAllowedRoots] = useState([]); + const [previewHeaders, setPreviewHeaders] = useState([]); + const [previewRows, setPreviewRows] = useState[]>([]); + const [previewTotal, setPreviewTotal] = useState(0); + const [saving, setSaving] = useState(false); + const [tempId, setTempId] = useState(null); + + useEffect(() => { + if (!open) return; + void (async () => { + try { + const roots = await FileReaderAPI.allowedRoots(); + setAllowedRoots(roots); + } catch {/* ignore */} + try { + const dbs = await ExternalDbConnectionAPI.getConnections(); + setDbList((dbs as any[]).map(d => ({ id: d.id, connection_name: d.connection_name }))); + } catch {/* ignore */} + if (editingId) { + try { + const { config, mappings: m } = await FileReaderAPI.get(editingId); + setCfg(config); + setMappings(m); + setTempId(editingId); + } catch (e) { + toast({ title: "조회 실패", description: (e as Error).message, variant: "destructive" }); + } + } else { + setCfg(defaultConfig); + setMappings([]); + setTempId(null); + } + setPreviewHeaders([]); + setPreviewRows([]); + setPreviewTotal(0); + })(); + }, [open, editingId]); + + const addMapping = (fromCol?: string) => { + setMappings(m => [...m, { + from_column: fromCol || "", + from_column_type: "header", + to_column: fromCol || "", + to_data_type: "TEXT", + is_required: false, + mapping_order: m.length, + }]); + }; + + const updateMapping = (i: number, patch: Partial) => { + setMappings(m => m.map((x, idx) => idx === i ? { ...x, ...patch } : x)); + }; + + const removeMapping = (i: number) => { + setMappings(m => m.filter((_, idx) => idx !== i)); + }; + + const handlePreview = async () => { + // 미리보기는 저장된 config 필요 — 임시 저장 후 preview + let id = tempId; + if (!id) { + try { + id = await FileReaderAPI.create({ ...cfg, name: cfg.name || `temp_${Date.now()}` }); + setTempId(id); + } catch (e) { + toast({ title: "임시 저장 실패", description: (e as Error).message, variant: "destructive" }); + return; + } + } else { + try { + await FileReaderAPI.update(id, cfg); + } catch (e) { + toast({ title: "설정 동기화 실패", description: (e as Error).message, variant: "destructive" }); + return; + } + } + const inp = document.createElement("input"); + inp.type = "file"; + inp.accept = ".csv,.tsv,.txt,.xlsx,.xls"; + inp.onchange = async () => { + if (!inp.files || inp.files.length === 0) return; + try { + const r = await FileReaderAPI.preview(id!, inp.files[0]); + setPreviewHeaders(r.headers); + setPreviewRows(r.sampleRows); + setPreviewTotal(r.totalRows); + toast({ title: "미리보기", description: `총 ${r.totalRows}행 / 헤더 ${r.headers.length}개` }); + } catch (e) { + toast({ title: "미리보기 실패", description: (e as Error).message, variant: "destructive" }); + } + }; + inp.click(); + }; + + const handleSave = async () => { + if (!cfg.name) { + toast({ title: "이름 필수", variant: "destructive" }); + return; + } + setSaving(true); + try { + const payload = { ...cfg, mappings }; + if (tempId) { + await FileReaderAPI.update(tempId, payload); + } else { + await FileReaderAPI.create(payload); + } + toast({ title: "저장 완료" }); + onSaved(); + } catch (e) { + toast({ title: "저장 실패", description: (e as Error).message, variant: "destructive" }); + } finally { + setSaving(false); + } + }; + + return ( + !o && onClose()}> + + + {editingId ? "파일 리더 수정" : "새 파일 리더"} + + + {/* 기본 정보 */} +
+
+ + setCfg(c => ({ ...c, name: e.target.value }))} /> +
+
+ + +
+ +
+ + +
+
+ setCfg(c => ({ ...c, is_active: v ? "Y" : "N" }))} /> + 활성 +
+ + {cfg.source_mode === "watch" && ( + <> +
+ + setCfg(c => ({ ...c, host_path: e.target.value }))} /> +
+ 허용 루트: {allowedRoots.length ? allowedRoots.join(", ") : "(로드 중)"} +
+
+
+ + setCfg(c => ({ ...c, file_pattern: e.target.value }))} /> +
+
+ + setCfg(c => ({ ...c, cron_schedule: e.target.value }))} /> +
+
+ + +
+
+ + setCfg(c => ({ ...c, archive_subdir: e.target.value }))} /> +
+ + )} + + {/* 파싱 옵션 */} +
+ setCfg(c => ({ ...c, has_header: v }))} /> + 첫 행이 헤더 +
+
+ + setCfg(c => ({ ...c, delimiter: e.target.value }))} /> +
+
+ + setCfg(c => ({ ...c, encoding: e.target.value }))} /> +
+
+ + setCfg(c => ({ ...c, skip_rows: Number(e.target.value) }))} /> +
+ {(cfg.file_type === "xlsx" || cfg.file_type === "xls") && ( +
+ + setCfg(c => ({ ...c, sheet_name: e.target.value }))} /> +
+ )} + + {/* 타겟 DB */} +
+ + +
+
+ + +
+
+ + setCfg(c => ({ ...c, target_schema: e.target.value }))} /> +
+
+ + setCfg(c => ({ ...c, target_table: e.target.value }))} /> +
+ {cfg.save_mode === "UPSERT" && ( +
+ + setCfg(c => ({ ...c, conflict_keys: e.target.value }))} /> +
+ )} +
+ +