From cb1a907007644a29e1bfa646c35aa64a6d669db5 Mon Sep 17 00:00:00 2001 From: ma-zhongxu Date: Tue, 4 Mar 2025 19:46:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0=E5=92=8C?= =?UTF-8?q?=E6=94=BF=E7=AD=96=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- collector/index.js | 5 +- frontend/src/pages/PolicyLibrary/index.jsx | 19 +- server/endpoints/system.js | 22 +- server/endpoints/workspaces.js | 168 ++++++------- server/models/documents.js | 4 + server/models/workspace.js | 1 + server/utils/files/index.js | 260 ++++++++++++++++----- 7 files changed, 332 insertions(+), 147 deletions(-) diff --git a/collector/index.js b/collector/index.js index 99bdf24..8f3edab 100644 --- a/collector/index.js +++ b/collector/index.js @@ -69,6 +69,9 @@ app.post( console.log("文件名:", filename); try { + const targetFilename = path + .normalize(filename) + .replace(/^(\.\.(\/|\\|$))+/, ""); const inputPath = path.resolve("./hotdir"); const sourceFile = path.join(inputPath, filename); // 拼接文件路径 console.log("源文件路径:", sourceFile); @@ -87,7 +90,7 @@ app.post( const fileContentBase64 = fileContent.toString("base64"); // 将文件内容转换为 Base64 字符串 // 处理文件并返回结果 - const { success, reason, documents = [] } = await processSingleFile(sourceFile, options); + const { success, reason, documents = [] } = await processSingleFile(targetFilename, options); response.status(200).json({ filename: sourceFile, diff --git a/frontend/src/pages/PolicyLibrary/index.jsx b/frontend/src/pages/PolicyLibrary/index.jsx index 6ce855d..2383e59 100644 --- a/frontend/src/pages/PolicyLibrary/index.jsx +++ b/frontend/src/pages/PolicyLibrary/index.jsx @@ -48,9 +48,12 @@ function Directory({ useEffect(() => { async function fetchUsers() { const nodata = await System.localFiles(); - const emo = nodata.items[0].items - setFDataTo(emo) - console.log(2222,emo); + const em0 = nodata.items[0].items + const em1 = nodata.items[1].items + const em2 = nodata.items[2].items + const em = [...em0,...em1,...em2] + setFDataTo(em) + console.log(2222,nodata); } fetchUsers(); }, []); @@ -140,10 +143,10 @@ function Directory({ } return item.items.every((file) => selectedItems[file.id]); } - + return !!selectedItems[id]; }; - + const moveToFolder = async (folder) => { const toMove = []; @@ -186,7 +189,7 @@ function Directory({ setSearchTerm(searchValue); }, 500); - const filteredFiles =filterFileSearchResults(files, searchTerm) + const filteredFiles =filterFileSearchResults(files, searchTerm) // console.log(114545,filteredFiles); const handleContextMenu = (event) => { @@ -203,11 +206,11 @@ function Directory({ setShow(!show) console.log(show); } - + // 返回首页 const bindHome = () =>{ window.location = '/' - } + } return ( <> diff --git a/server/endpoints/system.js b/server/endpoints/system.js index b579192..9d4af42 100644 --- a/server/endpoints/system.js +++ b/server/endpoints/system.js @@ -56,6 +56,7 @@ const { } = require("../utils/middleware/chatHistoryViewable"); const { simpleSSOEnabled } = require("../utils/middleware/simpleSSOEnabled"); const { TemporaryAuthToken } = require("../models/temporaryAuthToken"); +const { DeptUsers } = require("../models/deptUsers"); function systemEndpoints(app) { if (!app) return; @@ -420,12 +421,29 @@ function systemEndpoints(app) { } ); + // app.get( + // "/system/local-files", + // [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])], + // async (_, response) => { + // try { + // const localFiles = await viewLocalFiles(); + // response.status(200).json({ localFiles }); + // } catch (e) { + // console.error(e.message, e); + // response.sendStatus(500).end(); + // } + // } + // ); + app.get( "/system/local-files", [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])], - async (_, response) => { + async (request, response) => { + const currUser = await userFromSession(request, response); + const deptusers = DeptUsers.get({ userId: currUser.id }); + const deptId = (await deptusers).deptUser.deptId; try { - const localFiles = await viewLocalFiles(); + const localFiles = await viewLocalFiles(deptId); response.status(200).json({ localFiles }); } catch (e) { console.error(e.message, e); diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js index 374a16a..f39b56c 100644 --- a/server/endpoints/workspaces.js +++ b/server/endpoints/workspaces.js @@ -162,87 +162,87 @@ function workspaceEndpoints(app) { // } // ); - app.post( - "/workspace/:slug/upload", - [ - validatedRequest, - flexUserRoleValid([ROLES.admin, ROLES.manager]), - handleFileUpload, - ], - async function (request, response) { - try { - const user = await userFromSession(request, response); - const deptUserRecord = await DeptUsers.get({ userId: user.id }); - if (!deptUserRecord.deptUser) { - return response.status(500).json({ success: false, error: "没有发现用户组织机构" }); - } - const Collector = new CollectorApi(); - const { originalname } = request.file; - const processingOnline = await Collector.online(); - - if (!processingOnline) { - response - .status(500) - .json({ - success: false, - error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`, - }) - .end(); - return; - } - - const { success, reason, documents } = - await Collector.processDocument(originalname); - if (!success) { - response.status(500).json({ success: false, error: reason }).end(); - return; - } - // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - // 假设路径字符串 - const location = documents[0].location; - // 将路径中的反斜杠替换为正斜杠（可选，但通常更通用） - const unixStylePath = location.replace(/\\/g, '/'); - // 找到最后一个目录分隔符的位置 - const lastIndex = unixStylePath.lastIndexOf('/'); - // 提取文件名 - const parsedFileName = unixStylePath.substring(lastIndex + 1); - const fileExtension = path.extname(request.file.path).toLowerCase(); - const sourceFile = path.resolve(__dirname, request.file.destination, request.file.originalname); - const targetDir = - process.env.NODE_ENV === "development" - ? path.resolve(__dirname, `../../server/storage/localFile`) - : path.resolve(process.env.STORAGE_DIR, `../../server/storage/localFile`); - const newFileName = uuidv4() + fileExtension; // 新文件名 - moveAndRenameFile(sourceFile, targetDir, newFileName); - const deptDocData = { - deptId: deptUserRecord.deptUser.deptId, - parsedFileName: parsedFileName, - parsedFilePath: location, - realFileName: originalname, - realFileAlias: newFileName, - realFilePath: targetDir, - }; - await DeptDocument.create(deptDocData); - // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - Collector.log( - `Document ${originalname} uploaded processed and successfully. It is now available in documents.` - ); - await Telemetry.sendTelemetry("document_uploaded"); - await EventLogs.logEvent( - "document_uploaded", - { - documentName: originalname, - }, - response.locals?.user?.id - ); - response.status(200).json({ success: true, error: null }); - } catch (e) { - console.error(e.message, e); - response.sendStatus(500).end(); - } - } - ); + // app.post( + // "/workspace/:slug/upload", + // [ + // validatedRequest, + // flexUserRoleValid([ROLES.admin, ROLES.manager]), + // handleFileUpload, + // ], + // async function (request, response) { + // try { + // const user = await userFromSession(request, response); + // const deptUserRecord = await DeptUsers.get({ userId: user.id }); + // if (!deptUserRecord.deptUser) { + // return response.status(500).json({ success: false, error: "没有发现用户组织机构" }); + // } + // const Collector = new CollectorApi(); + // const { originalname } = request.file; + // const processingOnline = await Collector.online(); + // + // if (!processingOnline) { + // response + // .status(500) + // .json({ + // success: false, + // error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`, + // }) + // .end(); + // return; + // } + // + // const { success, reason, documents } = + // await Collector.processDocument(originalname); + // if (!success) { + // response.status(500).json({ success: false, error: reason }).end(); + // return; + // } + // // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + // // 假设路径字符串 + // const location = documents[0].location; + // // 将路径中的反斜杠替换为正斜杠（可选，但通常更通用） + // const unixStylePath = location.replace(/\\/g, '/'); + // // 找到最后一个目录分隔符的位置 + // const lastIndex = unixStylePath.lastIndexOf('/'); + // // 提取文件名 + // const parsedFileName = unixStylePath.substring(lastIndex + 1); + // const fileExtension = path.extname(request.file.path).toLowerCase(); + // const sourceFile = path.resolve(__dirname, request.file.destination, request.file.originalname); + // const targetDir = + // process.env.NODE_ENV === "development" + // ? path.resolve(__dirname, `../../server/storage/localFile`) + // : path.resolve(process.env.STORAGE_DIR, `../../server/storage/localFile`); + // const newFileName = uuidv4() + fileExtension; // 新文件名 + // moveAndRenameFile(sourceFile, targetDir, newFileName); + // const deptDocData = { + // deptId: deptUserRecord.deptUser.deptId, + // parsedFileName: parsedFileName, + // parsedFilePath: location, + // realFileName: originalname, + // realFileAlias: newFileName, + // realFilePath: targetDir, + // }; + // await DeptDocument.create(deptDocData); + // // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + // + // Collector.log( + // `Document ${originalname} uploaded processed and successfully. It is now available in documents.` + // ); + // await Telemetry.sendTelemetry("document_uploaded"); + // await EventLogs.logEvent( + // "document_uploaded", + // { + // documentName: originalname, + // }, + // response.locals?.user?.id + // ); + // response.status(200).json({ success: true, error: null }); + // } catch (e) { + // console.error(e.message, e); + // response.sendStatus(500).end(); + // } + // } + // ); @@ -255,6 +255,7 @@ function workspaceEndpoints(app) { ], async function (request, response) { try { + const user = await userFromSession(request, response); const deptUserRecord = await DeptUsers.get({ userId: user.id }); if (!deptUserRecord.deptUser) { return response.status(500).json({ success: false, error: "没有发现用户组织机构" }); @@ -314,6 +315,7 @@ function workspaceEndpoints(app) { realFileName: originalname, realFileAlias: newFileName, realFilePath: targetDir, + // parsedFileId: targetDir, }; await DeptDocument.create(deptDocData); // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -395,6 +397,8 @@ function workspaceEndpoints(app) { ? await Workspace.getWithUser(user, { slug }) : await Workspace.get({ slug }); + console.log("adds===============", adds); + if (!currWorkspace) { response.sendStatus(400).end(); return; @@ -539,7 +543,7 @@ function workspaceEndpoints(app) { const workspace = multiUserMode(response) ? await Workspace.getWithUser(user, { slug }) : await Workspace.get({ slug }); - + console.log("workspace++++++++++++++++++++++++++++", workspace); response.status(200).json({ workspace }); } catch (e) { console.error(e.message, e); diff --git a/server/models/documents.js b/server/models/documents.js index 81c2dd9..c3c494a 100644 --- a/server/models/documents.js +++ b/server/models/documents.js @@ -89,6 +89,8 @@ const Document = { for (const path of additions) { const data = await fileData(path); + console.log("data+++++++++++++++++++++++++++++++++", data); + console.log("path.split(\"/\")+++++++++++++++++++++++++++++++++", path.split("/")); if (!data) continue; const docId = uuidv4(); @@ -117,6 +119,8 @@ const Document = { continue; } + console.log("newDoc+++++++++++++++++++++++++++++++++", newDoc); + try { await prisma.workspace_documents.create({ data: newDoc }); embedded.push(path); diff --git a/server/models/workspace.js b/server/models/workspace.js index bdfbda2..f5b8593 100644 --- a/server/models/workspace.js +++ b/server/models/workspace.js @@ -278,6 +278,7 @@ const Workspace = { documents: true, }, }); + console.log("0000000000000000000000000000000000000000000000000000000000000000",workspace); return workspace || null; } catch (error) { diff --git a/server/utils/files/index.js b/server/utils/files/index.js index a28c2b4..48e8938 100644 --- a/server/utils/files/index.js +++ b/server/utils/files/index.js @@ -3,6 +3,8 @@ const path = require("path"); const { v5: uuidv5 } = require("uuid"); const { Document } = require("../../models/documents"); const { DocumentSyncQueue } = require("../../models/documentSyncQueue"); +const { userFromSession } = require("../http"); +const { DeptDocument } = require("../../models/deptDocument"); const documentsPath = process.env.NODE_ENV === "development" ? path.resolve(__dirname, `../../storage/documents`) @@ -24,71 +26,221 @@ async function fileData(filePath = null) { return JSON.parse(data); } -async function viewLocalFiles() { - if (!fs.existsSync(documentsPath)) fs.mkdirSync(documentsPath); - const liveSyncAvailable = await DocumentSyncQueue.enabled(); +// async function viewLocalFiles() { +// if (!fs.existsSync(documentsPath)) fs.mkdirSync(documentsPath); +// const liveSyncAvailable = await DocumentSyncQueue.enabled(); +// const directory = { +// name: "documents", +// type: "folder", +// items: [], +// }; +// for (const file of fs.readdirSync(documentsPath)) { +// // console.log("file:", file); +// if (path.extname(file) === ".md") continue; +// const folderPath = path.resolve(documentsPath, file); +// const isFolder = fs.lstatSync(folderPath).isDirectory(); +// if (isFolder) { +// const subdocs = { +// name: file, +// type: "folder", +// items: [], +// }; +// const subfiles = fs.readdirSync(folderPath); +// const filenames = {}; +// for (const subfile of subfiles) { +// if (path.extname(subfile) !== ".json") continue; +// const filePath = path.join(folderPath, subfile); +// const rawData = fs.readFileSync(filePath, "utf8"); +// // console.log("rawData:", rawData); +// const cachefilename = `${file}/${subfile}`; +// const { pageContent, ...metadata } = JSON.parse(rawData); +// subdocs.items.push({ +// name: subfile, +// type: "file", +// ...metadata, +// cached: await cachedVectorInformation(cachefilename, true), +// canWatch: liveSyncAvailable +// ? DocumentSyncQueue.canWatch(metadata) +// : false, +// // pinnedWorkspaces: [], // This is the list of workspaceIds that have pinned this document +// // watched: false, // boolean to indicate if this document is watched in ANY workspace +// }); +// filenames[cachefilename] = subfile; +// } +// +// // Grab the pinned workspaces and watched documents for this folder's documents +// // at the time of the query so we don't have to re-query the database for each file +// const pinnedWorkspacesByDocument = +// await getPinnedWorkspacesByDocument(filenames); +// const watchedDocumentsFilenames = +// await getWatchedDocumentFilenames(filenames); +// for (const item of subdocs.items) { +// item.pinnedWorkspaces = pinnedWorkspacesByDocument[item.name] || []; +// item.watched = +// watchedDocumentsFilenames.hasOwnProperty(item.name) || false; +// } +// +// directory.items.push(subdocs); +// } +// } +// +// // Make sure custom-documents is always the first folder in picker +// directory.items = [ +// directory.items.find((folder) => folder.name === "custom-documents"), +// ...directory.items.filter((folder) => folder.name !== "custom-documents"), +// ].filter((i) => !!i); +// +// return directory; +// } + +// async function viewLocalFiles(deptId) { +// const directory = { +// name: "documents", +// type: "folder", +// items: [], +// }; +// if (!fs.existsSync(documentsPath)) fs.mkdirSync(documentsPath); +// const liveSyncAvailable = await DocumentSyncQueue.enabled(); +// +// // 查询 deptDocuments +// const deptDocuments = await DeptDocument.where({ deptId: deptId, delTag: false }); +// if (!deptDocuments || deptDocuments.length === 0) { +// return directory; +// } +// +// // 初始化分类对象 +// const publicd = { +// name: "公开", +// type: "folder", +// items: [], +// }; +// const privated = { +// name: "私有", +// type: "folder", +// items: [], +// }; +// const temp = { +// name: "临时", +// type: "folder", +// items: [], +// }; +// +// // 遍历 deptDocuments +// for (const doc of deptDocuments) { +// try { +// const filePath = doc.parsedFilePath; // 获取文件路径 +// if (!fs.existsSync(filePath)) continue; // 如果文件不存在，跳过 +// +// // 读取文件内容 +// const rawData = fs.readFileSync(filePath, 'utf8'); +// const { pageContent, ...metadata } = JSON.parse(rawData); +// +// // 构造文件信息对象（保持与原方法一致的字段） +// const fileInfo = { +// name: path.basename(filePath), // 文件名 +// type: "file", +// ...metadata, +// cached: await cachedVectorInformation(filePath, true), +// canWatch: liveSyncAvailable +// ? DocumentSyncQueue.canWatch(metadata) +// : false, +// pinnedWorkspaces: [], // 初始化为空数组 +// watched: false, // 初始化为 false +// }; +// +// // 根据 isPublic 属性分类 +// if (doc.isPublic === 0) { +// publicd.items.push(fileInfo); +// } else if (doc.isPublic === 1) { +// privated.items.push(fileInfo); +// } else { +// temp.items.push(fileInfo); +// } +// } catch (error) { +// console.error(`Error processing file ${doc.parsedFilePath}:`, error); +// } +// } +// +// directory.items = [publicd, privated, temp]; +// // 返回嵌套结构 +// return directory; +// } + + +async function viewLocalFiles(deptId) { const directory = { name: "documents", type: "folder", items: [], }; - // console.log("111111111111111111111111111111111111111111111111111111111111111111111111111111111"); - for (const file of fs.readdirSync(documentsPath)) { - // console.log("file:", file); - if (path.extname(file) === ".md") continue; - const folderPath = path.resolve(documentsPath, file); - const isFolder = fs.lstatSync(folderPath).isDirectory(); - if (isFolder) { - const subdocs = { - name: file, - type: "folder", - items: [], + if (!fs.existsSync(documentsPath)) fs.mkdirSync(documentsPath); + const liveSyncAvailable = await DocumentSyncQueue.enabled(); + + // 查询 deptDocuments + const deptDocuments = await DeptDocument.where({ deptId: deptId, delTag: false }); + if (!deptDocuments || deptDocuments.length === 0) { + return directory; + } + + // 初始化分类对象 + const publicd = { + name: "公开", + type: "folder", + items: [], + }; + const privated = { + name: "私有", + type: "folder", + items: [], + }; + const temp = { + name: "临时", + type: "folder", + items: [], + }; + + // 遍历 deptDocuments + for (const doc of deptDocuments) { + try { + const filePath = doc.parsedFilePath; // 获取文件路径 + if (!fs.existsSync(filePath)) continue; // 如果文件不存在，跳过 + + // 读取文件内容 + const rawData = fs.readFileSync(filePath, 'utf8'); + const { pageContent, ...metadata } = JSON.parse(rawData); + + // 计算相对路径，并将路径分隔符统一为 `/` + const relativePath = path.relative(documentsPath, filePath).replace(/\\/g, '/'); + + // 构造文件信息对象（保持与原方法一致的字段） + const fileInfo = { + name: path.basename(filePath), // 文件名 + type: "file", + ...metadata, + cached: await cachedVectorInformation(filePath, true), + canWatch: liveSyncAvailable + ? DocumentSyncQueue.canWatch(metadata) + : false, + pinnedWorkspaces: [], // 初始化为空数组 + watched: false, // 初始化为 false + relativePath: relativePath, // 新增字段：相对路径（使用 `/` 分隔符） }; - const subfiles = fs.readdirSync(folderPath); - const filenames = {}; - for (const subfile of subfiles) { - if (path.extname(subfile) !== ".json") continue; - const filePath = path.join(folderPath, subfile); - const rawData = fs.readFileSync(filePath, "utf8"); - // console.log("rawData:", rawData); - const cachefilename = `${file}/${subfile}`; - const { pageContent, ...metadata } = JSON.parse(rawData); - subdocs.items.push({ - name: subfile, - type: "file", - ...metadata, - cached: await cachedVectorInformation(cachefilename, true), - canWatch: liveSyncAvailable - ? DocumentSyncQueue.canWatch(metadata) - : false, - // pinnedWorkspaces: [], // This is the list of workspaceIds that have pinned this document - // watched: false, // boolean to indicate if this document is watched in ANY workspace - }); - filenames[cachefilename] = subfile; - } - // Grab the pinned workspaces and watched documents for this folder's documents - // at the time of the query so we don't have to re-query the database for each file - const pinnedWorkspacesByDocument = - await getPinnedWorkspacesByDocument(filenames); - const watchedDocumentsFilenames = - await getWatchedDocumentFilenames(filenames); - for (const item of subdocs.items) { - item.pinnedWorkspaces = pinnedWorkspacesByDocument[item.name] || []; - item.watched = - watchedDocumentsFilenames.hasOwnProperty(item.name) || false; + // 根据 isPublic 属性分类 + if (doc.isPublic === 0) { + publicd.items.push(fileInfo); + } else if (doc.isPublic === 1) { + privated.items.push(fileInfo); + } else { + temp.items.push(fileInfo); } - - directory.items.push(subdocs); + } catch (error) { + console.error(`Error processing file ${doc.parsedFilePath}:`, error); } } - // Make sure custom-documents is always the first folder in picker - directory.items = [ - directory.items.find((folder) => folder.name === "custom-documents"), - ...directory.items.filter((folder) => folder.name !== "custom-documents"), - ].filter((i) => !!i); - + directory.items = [publicd, privated, temp]; + // 返回嵌套结构 return directory; }