diff --git a/backend/package-lock.json b/backend/package-lock.json index 90ebad5..ac716ca 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,6 +15,7 @@ "@types/nodemailer": "^7.0.2", "@types/qrcode": "^1.5.5", "@types/speakeasy": "^2.0.10", + "adm-zip": "^0.5.17", "bcryptjs": "^2.4.3", "cors": "^2.8.5", "dotenv": "^16.3.1", @@ -32,6 +33,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@types/adm-zip": "^0.5.8", "@types/bcryptjs": "^2.4.6", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", @@ -4968,6 +4970,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/adm-zip": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.8.tgz", + "integrity": "sha512-RVVH7QvZYbN+ihqZ4kX/dMiowf6o+Jk1fNwiSdx0NahBJLU787zkULhGhJM8mf/obmLGmgdMM0bXsQTmyfbR7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/aws-lambda": { "version": "8.10.152", "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.152.tgz", @@ -5327,6 +5339,15 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.17", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.17.tgz", + "integrity": "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", diff --git a/backend/package.json b/backend/package.json index 5cfb43e..3f751ba 100644 --- a/backend/package.json +++ b/backend/package.json @@ -20,6 +20,7 @@ "@types/nodemailer": "^7.0.2", "@types/qrcode": "^1.5.5", "@types/speakeasy": "^2.0.10", + "adm-zip": "^0.5.17", "bcryptjs": "^2.4.3", "cors": "^2.8.5", "dotenv": "^16.3.1", @@ -37,6 +38,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@types/adm-zip": "^0.5.8", "@types/bcryptjs": "^2.4.6", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", diff --git a/backend/src/routes/crc.ts b/backend/src/routes/crc.ts index 8797649..236f85f 100644 --- a/backend/src/routes/crc.ts +++ b/backend/src/routes/crc.ts @@ -9,6 +9,7 @@ import { recordEvent } from "../services/auditLogService"; import fs from "fs"; import path from "path"; import multer from "multer"; +import AdmZip from "adm-zip"; import { syncRiskFromResponse } from "../services/crcRiskService"; import crypto from "crypto"; import { inngest } from "../inngest/client"; @@ -66,6 +67,32 @@ const templateUpload = multer({ }, }); +const bulkTemplateUpload = multer({ + storage: templateStorage, + limits: { + fileSize: 25 * 1024 * 1024, // 25MB max per file + files: 160, // Max 160 files per request + }, + fileFilter: (_req, file, cb) => { + const allowed = [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", // .docx + "application/msword", // .doc + "application/zip", // .zip + "application/x-zip-compressed", // .zip (Windows/some systems) + ]; + if ( + allowed.includes(file.mimetype) || + file.originalname.endsWith(".docx") || + file.originalname.endsWith(".doc") || + file.originalname.endsWith(".zip") + ) { + cb(null, true); + } else { + cb(new Error("Only .doc, .docx, and .zip files are allowed")); + } + }, +}); + // --- Validation Schemas --- // Common reusable schemas @@ -1808,6 +1835,307 @@ router.get("/templates/status", authenticateToken, requireRole(["ADMIN"]), async } }); +// POST /crc/templates/bulk-upload - Bulk upload compliance template documents (Admin only) +router.post("/templates/bulk-upload", authenticateToken, requireRole(["ADMIN"]), (req, res, next) => { + bulkTemplateUpload.array("files", 140)(req, res, (err: any) => { + if (err instanceof multer.MulterError) { + if (err.code === "LIMIT_FILE_SIZE") { + return res.status(400).json({ success: false, error: "File size exceeds 25MB limit" }); + } + return res.status(400).json({ success: false, error: err.message }); + } + if (err) { + return res.status(400).json({ success: false, error: err.message }); + } + next(); + }); +}, async (req, res) => { + const uploadedFiles = req.files as Express.Multer.File[] | undefined; + if (!uploadedFiles || uploadedFiles.length === 0) { + return res.status(400).json({ success: false, error: "No files uploaded." }); + } + + // Temporary folders created for extracting ZIP files + const tempFolders: string[] = []; + // All template files to process (either direct uploads or extracted from ZIPs) + const templatesToProcess: { filePath: string; originalname: string; mimetype: string; size: number; isTemp: boolean }[] = []; + + try { + // 1. Fetch all valid control IDs from database to match against filenames + const controlsResult = await pool.query("SELECT control_id FROM crc_controls"); + const validControlIds = controlsResult.rows.map((r: { control_id: string }) => r.control_id); + + // Sort control IDs by length in descending order to match the longest (most specific) first + // E.g., match "GOV-3P-010" before "GOV-3P-01" + validControlIds.sort((a, b) => b.length - a.length); + + // Cumulative safety counters across all ZIPs in the request + let totalExtractedFileCount = 0; + let totalExtractedBytes = 0; + + // 2. Classify and process each uploaded file + for (const file of uploadedFiles) { + const isZip = file.mimetype === "application/zip" || + file.mimetype === "application/x-zip-compressed" || + file.originalname.toLowerCase().endsWith(".zip"); + + if (isZip) { + // Extract ZIP contents + const zipPath = file.path; + const extractDir = path.join(TEMPLATES_DIR, `bulk_extract_${Date.now()}_${crypto.randomUUID()}`); + fs.mkdirSync(extractDir, { recursive: true }); + tempFolders.push(extractDir); + + const zip = new AdmZip(zipPath); + const entries = zip.getEntries(); + + // Safety limits to prevent ZIP bombs or excessive uncompressed extraction (cumulative) + const MAX_ZIP_ENTRIES = 150; + const MAX_ZIP_UNCOMPRESSED_SIZE = 100 * 1024 * 1024; // 100MB + + for (const entry of entries) { + if (!entry.isDirectory) { + totalExtractedFileCount++; + totalExtractedBytes += entry.header.size; + } + } + + if (totalExtractedFileCount > MAX_ZIP_ENTRIES) { + throw new Error(`Total files across all ZIP archives exceeds limit (max: ${MAX_ZIP_ENTRIES})`); + } + if (totalExtractedBytes > MAX_ZIP_UNCOMPRESSED_SIZE) { + throw new Error(`Total uncompressed size across all ZIP archives exceeds limit (max: 100MB)`); + } + + // Pre-scan, filter and extract only allowed .doc/.docx template files + for (const entry of entries) { + if (entry.isDirectory) continue; + + const nameLower = entry.name.toLowerCase(); + const isAllowedTemplate = nameLower.endsWith(".docx") || nameLower.endsWith(".doc"); + + if (isAllowedTemplate) { + zip.extractEntryTo(entry, extractDir, true, true); + } + } + + // Recursively read extracted files + const readFilesRecursively = (dir: string) => { + const entriesInDir = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entriesInDir) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + readFilesRecursively(fullPath); + } else if (entry.isFile() && (entry.name.toLowerCase().endsWith(".docx") || entry.name.toLowerCase().endsWith(".doc"))) { + const stats = fs.statSync(fullPath); + const isDoc = entry.name.toLowerCase().endsWith(".doc"); + templatesToProcess.push({ + filePath: fullPath, + originalname: entry.name, + mimetype: isDoc ? "application/msword" : "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + size: stats.size, + isTemp: true, // We need to delete this file later because it was extracted + }); + } + } + }; + readFilesRecursively(extractDir); + } else { + // Direct .docx / .doc file + templatesToProcess.push({ + filePath: file.path, + originalname: file.originalname, + mimetype: file.mimetype, + size: file.size, + isTemp: false, // Will be deleted at the end as part of standard multer file cleanups + }); + } + } + + if (templatesToProcess.length === 0) { + return res.status(400).json({ success: false, error: "No valid .doc or .docx template files found in the upload." }); + } + + // 3. Precompute and validate template-to-control ID matches + // Require bounded token-style matches and reject any ambiguous or duplicate template-to-control assignments + const templateMappings = new Map(); + const validationErrors: string[] = []; + const matchedControlIdsForTemplates = new Map(); // index in templatesToProcess -> control_id + + for (let i = 0; i < templatesToProcess.length; i++) { + const template = templatesToProcess[i]; + const matchedControls: string[] = []; + + for (const controlId of validControlIds) { + // Bounded token-style match to prevent matching partial substrings (e.g. matching GOV-3P-01 inside GOV-3P-010) + const escaped = controlId.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); + const regex = new RegExp(`(?:^|[^A-Z0-9])${escaped}(?:$|[^A-Z0-9])`, 'i'); + if (regex.test(template.originalname)) { + matchedControls.push(controlId); + } + } + + if (matchedControls.length > 1) { + validationErrors.push(`Ambiguous match: File "${template.originalname}" matches multiple control IDs (${matchedControls.join(", ")})`); + } else if (matchedControls.length === 1) { + const matchedId = matchedControls[0]; + const existing = templateMappings.get(matchedId); + if (existing) { + validationErrors.push(`Duplicate template assignment: Both "${existing.filename}" and "${template.originalname}" map to control ID "${matchedId}"`); + } else { + templateMappings.set(matchedId, { filename: template.originalname, templateIdx: i }); + matchedControlIdsForTemplates.set(i, matchedId); + } + } + // If matchedControls.length === 0, it is unmatched, which we report as unmatched in the response details list + } + + if (validationErrors.length > 0) { + return res.status(400).json({ + success: false, + error: "Validation failed: " + validationErrors.join("; ") + }); + } + + // 4. Process each template file after passing pre-validation + const results: { filename: string; status: "success" | "unmatched" | "failed"; controlId?: string; error?: string }[] = []; + const FileConstructor = typeof File !== "undefined" ? File : require("node:buffer").File; + const user = (req as any).user; + + for (let i = 0; i < templatesToProcess.length; i++) { + const template = templatesToProcess[i]; + const matchedControlId = matchedControlIdsForTemplates.get(i); + + if (!matchedControlId) { + results.push({ + filename: template.originalname, + status: "unmatched", + error: "Filename does not match any valid control ID in the database.", + }); + continue; + } + + try { + const fileData = fs.readFileSync(template.filePath); + const webFile = new FileConstructor([fileData], template.originalname, { type: template.mimetype }); + + // Upload to UploadThing + const uploadResult = await utapi.uploadFiles(webFile); + + if (!uploadResult || !uploadResult.data) { + const errorMsg = (uploadResult as any).error?.message || "Failed to upload file to cloud storage"; + throw new Error(errorMsg); + } + + const { url, key: fileKey } = uploadResult.data; + + // Check for existing database record to clean up superseded files from UploadThing + const existingResult = await pool.query( + "SELECT file_key FROM crc_control_templates WHERE control_id = $1", + [matchedControlId] + ); + const oldKey = existingResult.rows.length > 0 ? existingResult.rows[0].file_key : null; + + // Save or update mapping in database + await pool.query( + `INSERT INTO crc_control_templates (control_id, url, file_key, filename, size, updated_at) + VALUES ($1, $2, $3, $4, $5, NOW()) + ON CONFLICT (control_id) + DO UPDATE SET url = $2, file_key = $3, filename = $4, size = $5, updated_at = NOW()`, + [matchedControlId, url, fileKey, template.originalname, template.size] + ); + + // Delete superseded file from UploadThing only AFTER database upsert succeeds + if (oldKey) { + try { + await utapi.deleteFiles(oldKey); + } catch (deleteErr) { + console.error(`Failed to delete superseded template from UploadThing: ${oldKey}`, deleteErr); + } + } + + // Try to remove local legacy files for this control to keep the local filesystem pristine + for (const legacyExt of [".docx", ".doc"] as const) { + try { + const legacyPath = resolveTemplatePath(matchedControlId, legacyExt); + if (fs.existsSync(legacyPath)) { + fs.unlinkSync(legacyPath); + } + } catch (resolveErr) { + console.error(`Skipping legacy file cleanup for ${matchedControlId} with extension ${legacyExt}:`, resolveErr); + } + } + + // Log audit log event separately wrapped in its own try/catch to not block the request outcome + try { + await recordEvent({ + projectId: null, + actorId: user?.id, + action: "UPLOAD_CRC_TEMPLATE", + objectType: "CRC_CONTROL", + objectId: matchedControlId, + metadata: { filename: template.originalname, source: "bulk-uploadthing" }, + }); + } catch (auditErr) { + console.error(`Failed to log audit event for template upload of control ${matchedControlId}:`, auditErr); + } + + results.push({ + filename: template.originalname, + status: "success", + controlId: matchedControlId, + }); + + } catch (uploadErr: any) { + console.error(`Error processing template ${template.originalname} for control ${matchedControlId}:`, uploadErr); + results.push({ + filename: template.originalname, + status: "failed", + controlId: matchedControlId, + error: uploadErr.message || "Failed during upload or database save", + }); + } + } + + const summary = { + total: templatesToProcess.length, + success: results.filter(r => r.status === "success").length, + unmatched: results.filter(r => r.status === "unmatched").length, + failed: results.filter(r => r.status === "failed").length, + }; + + res.json({ + success: true, + message: `Bulk template upload finished. Success: ${summary.success}, Unmatched: ${summary.unmatched}, Failed: ${summary.failed}`, + summary, + details: results, + }); + + } catch (error) { + console.error("Fatal error in bulk template upload:", error); + res.status(500).json({ success: false, error: "Fatal error processing bulk templates" }); + } finally { + // 4. Cleanup all temporary files and extracted directories + if (uploadedFiles) { + for (const file of uploadedFiles) { + if (fs.existsSync(file.path)) { + try { fs.unlinkSync(file.path); } catch {} + } + } + } + + for (const folder of tempFolders) { + if (fs.existsSync(folder)) { + try { + fs.rmSync(folder, { recursive: true, force: true }); + } catch (rmErr) { + console.error(`Failed to clean up temporary folder: ${folder}`, rmErr); + } + } + } + } +}); + export async function handleTemplateDownloadAutoflip(projectId: string, controlShortId: string, userId: string) { try { // 1. Find the control UUID using controlShortId (it could be control_id like 'OPS-INC-01') @@ -2346,22 +2674,30 @@ router.post("/templates/:controlId/upload", authenticateToken, requireRole(["ADM } // Try to remove local legacy files for this control to keep the local filesystem pristine - for (const legacyExt of [".docx", ".doc"]) { - const legacyPath = path.join(TEMPLATES_DIR, `${controlShortId}${legacyExt}`); - if (fs.existsSync(legacyPath)) { - try { fs.unlinkSync(legacyPath); } catch {} + for (const legacyExt of [".docx", ".doc"] as const) { + try { + const legacyPath = resolveTemplatePath(controlShortId, legacyExt); + if (fs.existsSync(legacyPath)) { + fs.unlinkSync(legacyPath); + } + } catch (resolveErr) { + console.error(`Skipping legacy file cleanup for ${controlShortId} with extension ${legacyExt}:`, resolveErr); } } const user = (req as any).user; - await recordEvent({ - projectId: null, - actorId: user?.id, - action: "UPLOAD_CRC_TEMPLATE", - objectType: "CRC_CONTROL", - objectId: controlShortId, - metadata: { filename: tmpFile.originalname, source: "uploadthing" }, - }); + try { + await recordEvent({ + projectId: null, + actorId: user?.id, + action: "UPLOAD_CRC_TEMPLATE", + objectType: "CRC_CONTROL", + objectId: controlShortId, + metadata: { filename: tmpFile.originalname, source: "uploadthing" }, + }); + } catch (auditErr) { + console.error(`Failed to log audit event for template upload of control ${controlShortId}:`, auditErr); + } res.json({ success: true, diff --git a/frontend/src/app/admin/crc/page.tsx b/frontend/src/app/admin/crc/page.tsx index 96b3725..7d22b00 100644 --- a/frontend/src/app/admin/crc/page.tsx +++ b/frontend/src/app/admin/crc/page.tsx @@ -339,6 +339,15 @@ export default function CRCAdminPage() { const [templateUploadTargetId, setTemplateUploadTargetId] = useState(null); // UUID of control const [templateUploadTargetShortId, setTemplateUploadTargetShortId] = useState(null); // short control_id + // Template bulk upload state + const [showTemplateBulkDialog, setShowTemplateBulkDialog] = useState(false); + const [templateBulkFiles, setTemplateBulkFiles] = useState([]); + const [templateBulkUploading, setTemplateBulkUploading] = useState(false); + const [templateBulkResult, setTemplateBulkResult] = useState<{ + summary: { total: number; success: number; unmatched: number; failed: number }; + details: Array<{ filename: string; status: 'success' | 'unmatched' | 'failed'; controlId?: string; error?: string }>; + } | null>(null); + // Template delete dialog state const [showTemplateDeleteDialog, setShowTemplateDeleteDialog] = useState(false); const [templateToDeleteId, setTemplateToDeleteId] = useState(null); @@ -456,6 +465,41 @@ export default function CRCAdminPage() { } }; + const handleTemplateBulkFileChange = (e: React.ChangeEvent) => { + if (e.target.files) { + const files = Array.from(e.target.files); + setTemplateBulkFiles(prev => [...prev, ...files]); + e.target.value = ""; // Clear file input value to allow re-selecting the same file + } + }; + + const handleTemplateBulkRemoveFile = (index: number) => { + setTemplateBulkFiles(prev => prev.filter((_, i) => i !== index)); + }; + + const handleTemplateBulkUpload = async () => { + if (templateBulkFiles.length === 0 || templateBulkUploading) return; + setTemplateBulkUploading(true); + setTemplateBulkResult(null); + try { + const res = await apiService.uploadCRCTemplatesBulk(templateBulkFiles); + setTemplateBulkResult(res); + toast.success("Bulk template upload complete"); + fetchTemplateStatuses(); + } catch (error: any) { + toast.error(error.message || "Failed to bulk upload templates"); + } finally { + setTemplateBulkUploading(false); + } + }; + + const closeTemplateBulkDialog = () => { + setShowTemplateBulkDialog(false); + setTemplateBulkFiles([]); + setTemplateBulkResult(null); + setTemplateBulkUploading(false); + }; + useEffect(() => { const controller = new AbortController(); fetchCategories(controller.signal); @@ -1374,7 +1418,10 @@ export default function CRCAdminPage() { Manage Categories + + +
+ {templateBulkFiles.map((file, idx) => ( +
+
+ + {file.name} + ({(file.size / 1024).toFixed(1)} KB) +
+ +
+ ))} +
+ + )} + + + + + + + ) : ( +
+ {/* Summary Card */} +
+
+
{templateBulkResult.summary.total}
+
Processed
+
+
+
{templateBulkResult.summary.success}
+
Successful
+
+
+
{templateBulkResult.summary.unmatched}
+
Unmatched
+
+
+
{templateBulkResult.summary.failed}
+
Failed
+
+
+ + {/* Details List */} +
+

Upload Details

+
+ {templateBulkResult.details.map((detail, idx) => ( +
+
+ {detail.status === "success" ? ( + + ) : detail.status === "unmatched" ? ( + + ) : ( + + )} + {detail.filename} + {detail.controlId && ( + + {detail.controlId} + + )} +
+
+ {detail.status === "success" ? ( + Success + ) : detail.status === "unmatched" ? ( + Unmatched + ) : ( + Failed + )} +
+
+ ))} +
+
+ + + + +
+ )} + + ); } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ce97d24..d2d9a1e 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1789,6 +1789,34 @@ class ApiService { return response.json(); } + async uploadCRCTemplatesBulk(files: File[]): Promise<{ + success: boolean; + message: string; + summary: { total: number; success: number; unmatched: number; failed: number }; + details: Array<{ filename: string; status: 'success' | 'unmatched' | 'failed'; controlId?: string; error?: string }>; + }> { + const formData = new FormData(); + files.forEach((file) => { + formData.append("files", file); + }); + + const token = this.getAuthToken(); + const response = await fetch(`${API_BASE_URL}/crc/templates/bulk-upload`, { + method: "POST", + headers: { + ...(token && { Authorization: `Bearer ${token}` }), + }, + body: formData, + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({ error: "Bulk upload failed" })); + throw new Error(error.error || `HTTP ${response.status}`); + } + + return response.json(); + } + async deleteCRCTemplate(controlId: string): Promise<{ success: boolean; message: string }> { return this.request<{ success: boolean; message: string }>(`/crc/templates/${controlId}/template`, { method: "DELETE",