feat：Enhance RAG module with document processing and retrieval improvements (#395)

* feat: extend UniversalDocLoader to support additional file formats including Excel and HTML

* feat: implement RAG module with document loading, splitting, and processing capabilities

* feat: implement RAG module with document loading, splitting, and processing capabilities

* feat: update Milvus configuration and enhance file processing logic in RAG service

* feat: enhance RAG infrastructure with document processing, vector storage, and retrieval capabilities

* feat: add progress tracking for RAG file processing and enhance worker pool status management

* feat: enhance retrieval service with advanced search ranking and filtering capabilities

* feat: enhance retrieval service with BM25 indexing and improved ranking parameters

* feat: enhance retrieval service with BM25 indexing and improved ranking parameters

* feat: implement Milvus client singleton management and refactor vector store interactions

* feat: refactor file processing to use async session for database interactions

* feat: enhance chunk processing with filtering, cleaning, and batch storage

* feat: enhance API response models with additional fields and configuration

Author: Dallas98

.claude/skills/backend-architect/SKILL.md

@@ -97,7 +97,11 @@ const KnowledgeBaseDetailPage: React.FC = () => {
     handleKeywordChange,
   } = useFetchData<KBFile>(
     (params) => id ? queryKnowledgeBaseFilesUsingGet(id, params) : Promise.resolve({ data: [] }),
-    (file) => mapFileData(file, t)
+    (file) => mapFileData(file, t),
+    30000, // 30秒轮询间隔
+    false, // 不自动轮询
+    [], // 额外的轮询函数
+    0 // pageOffset: Python 后端期望 page 从 1 开始，前端 current=1 时传 page=1
   );
 
   // File table logic

.claude/skills/fastapi-templates/SKILL.md

@@ -31,7 +31,11 @@ export default function KnowledgeBasePage() {
     handleKeywordChange,
   } = useFetchData<KnowledgeBaseItem>(
     queryKnowledgeBasesUsingPost,
-    (kb) => mapKnowledgeBase(kb, false, t) // 在首页不显示索引模型和文本理解模型字段
+    (kb) => mapKnowledgeBase(kb, false, t), // 在首页不显示索引模型和文本理解模型字段
+    30000, // 30秒轮询间隔
+    false, // 不自动轮询
+    [], // 额外的轮询函数
+    0 // pageOffset: Python 后端期望 page 从 1 开始，前端 current=1 时传 page=1
   );
 
   useEffect(() => {

frontend/src/pages/KnowledgeBase/Detail/KnowledgeBaseDetail.tsx

@@ -1,8 +1,13 @@
 import { get, post, put, del } from "@/utils/request";
 
 // 获取知识库列表
-export function queryKnowledgeBasesUsingPost(params: object) {
-  return post("/api/knowledge-base/list", params);
+export function queryKnowledgeBasesUsingPost(params: any) {
+  // 将前端的 size 参数映射为后端的 page_size
+  const { size, ...rest } = params;
+  return post("/api/knowledge-base/list", {
+    ...rest,
+    page_size: size
+  });
 }
 
 // 创建知识库
@@ -26,8 +31,22 @@ export function deleteKnowledgeBaseByIdUsingDelete(baseId: string) {
 }
 
 // 获取知识生成文件列表
-export function queryKnowledgeBaseFilesUsingGet(baseId: string, params?: Record<string, string>) {
-  return get(`/api/knowledge-base/${baseId}/files${params ? `?${new URLSearchParams(params).toString()}` : ""}`);
+export function queryKnowledgeBaseFilesUsingGet(baseId: string, params?: Record<string, any>) {
+  if (!params) {
+    return get(`/api/knowledge-base/${baseId}/files`);
+  }
+  // 将前端的 size 参数映射为后端的 page_size
+  const { size, page, ...rest } = params;
+  const queryParams = {
+    page: page || 1,
+    page_size: size || 10,
+    ...rest
+  };
+  return get(`/api/knowledge-base/${baseId}/files?${new URLSearchParams(
+    Object.entries(queryParams)
+      .filter(([_, v]) => v !== undefined && v !== null)
+      .reduce((acc, [k, v]) => ({ ...acc, [k]: String(v) }), {})
+  ).toString()}`);
 }
 
 // 添加文件到知识库
@@ -62,5 +81,5 @@ export function queryKnowledgeBaseFileDetailUsingGet(
 ) {
   const page = params.page ?? 1;
   const size = params.size ?? 20;
-  return get(`/api/knowledge-base/${knowledgeBaseId}/files/${ragFileId}?page=${page}&size=${size}`);
+  return get(`/api/knowledge-base/${knowledgeBaseId}/files/${ragFileId}?page=${page}&page_size=${size}`);
 }

frontend/src/pages/KnowledgeBase/Home/KnowledgeBasePage.tsx

@@ -13,25 +13,54 @@ export default defineConfig({
   },
   server: {
     host: "0.0.0.0",
-    proxy: {
-      "^/api": {
-        target: "http://localhost:8080", // 本地后端服务地址
+    proxy: (() => {
+      const pythonProxyConfig = {
+        target: "http://localhost:18000",
         changeOrigin: true,
         secure: false,
-        rewrite: (path) => path.replace(/^\/api/, "/api"),
-        configure: (proxy, options) => {
-          // proxy 是 'http-proxy' 的实例
-          proxy.on("proxyReq", (proxyReq, req, res) => {
-            // 可以在这里修改请求头
-            proxyReq.removeHeader("referer");
-            proxyReq.removeHeader("origin");
+        configure: (proxy: { on: (event: string, handler: (arg: unknown) => void) => void }) => {
+          proxy.on("proxyReq", (proxyReq: unknown) => {
+            (proxyReq as { removeHeader: (name: string) => void }).removeHeader("referer");
+            (proxyReq as { removeHeader: (name: string) => void }).removeHeader("origin");
           });
-          proxy.on("proxyRes", (proxyRes, req, res) => {
-            delete proxyRes.headers["set-cookie"];
-            proxyRes.headers["cookies"] = ""; // 清除 cookies 头
+          proxy.on("proxyRes", (proxyRes: unknown) => {
+            const res = proxyRes as { headers: Record<string, unknown> };
+            delete res.headers["set-cookie"];
+            res.headers["cookies"] = "";
           });
         },
-      },
-    },
+      };
+
+      const javaProxyConfig = {
+        target: "http://localhost:8080",
+        changeOrigin: true,
+        secure: false,
+        configure: (proxy: { on: (event: string, handler: (arg: unknown) => void) => void }) => {
+          proxy.on("proxyReq", (proxyReq: unknown) => {
+            (proxyReq as { removeHeader: (name: string) => void }).removeHeader("referer");
+            (proxyReq as { removeHeader: (name: string) => void }).removeHeader("origin");
+          });
+          proxy.on("proxyRes", (proxyRes: unknown) => {
+            const res = proxyRes as { headers: Record<string, unknown> };
+            delete res.headers["set-cookie"];
+            res.headers["cookies"] = "";
+          });
+        },
+      };
+
+      // Python 服务: rag, synthesis, annotation, evaluation, models
+      const pythonPaths = ["rag", "synthesis", "annotation", "knowledge-base", "data-collection", "evaluation", "models"];
+      // Java 服务: data-management, knowledge-base
+      const javaPaths = ["data-management", "operators", "cleansing"];
+
+      const proxy: Record<string, object> = {};
+      for (const p of pythonPaths) {
+        proxy[`/api/${p}`] = pythonProxyConfig;
+      }
+      for (const p of javaPaths) {
+        proxy[`/api/${p}`] = javaProxyConfig;
+      }
+      return proxy;
+    })(),
   },
 });

frontend/src/pages/KnowledgeBase/knowledge-base.api.ts

@@ -77,5 +77,12 @@ def build_database_url(self):
 
     datamate_jwt_enable: bool = False
 
+    # Milvus 配置
+    milvus_uri: str = "http://localhost:19530"
+    milvus_token: str = ""
+
+    # 文件存储配置（共享文件系统）
+    file_storage_path: str = "/data/files"
+
 # 全局设置实例
 settings = Settings()

frontend/vite.config.ts

@@ -77,8 +77,17 @@ def __init__(self):
     # ========== RAG 模块 ==========
     RAG_CONFIG_ERROR: Final = ErrorCode("rag.0001", "RAG configuration error", 400)
     RAG_KNOWLEDGE_BASE_NOT_FOUND: Final = ErrorCode("rag.0002", "Knowledge base not found", 404)
-    RAG_MODEL_NOT_FOUND: Final = ErrorCode("rag.0003", "RAG model not found", 404)
-    RAG_QUERY_FAILED: Final = ErrorCode("rag.0004", "RAG query failed", 500)
+    RAG_KNOWLEDGE_BASE_ALREADY_EXISTS: Final = ErrorCode("rag.0003", "Knowledge base already exists", 400)
+    RAG_KNOWLEDGE_BASE_NAME_INVALID: Final = ErrorCode("rag.0004", "Knowledge base name is invalid", 400)
+    RAG_FILE_NOT_FOUND: Final = ErrorCode("rag.0005", "RAG file not found", 404)
+    RAG_FILE_PROCESS_FAILED: Final = ErrorCode("rag.0006", "File processing failed", 500)
+    RAG_FILE_PARSE_FAILED: Final = ErrorCode("rag.0007", "File parsing failed", 500)
+    RAG_CHUNK_NOT_FOUND: Final = ErrorCode("rag.0008", "Chunk not found", 404)
+    RAG_MODEL_NOT_FOUND: Final = ErrorCode("rag.0009", "RAG model not found", 404)
+    RAG_QUERY_FAILED: Final = ErrorCode("rag.0010", "RAG query failed", 500)
+    RAG_MILVUS_ERROR: Final = ErrorCode("rag.0011", "Milvus operation failed", 500)
+    RAG_COLLECTION_NOT_FOUND: Final = ErrorCode("rag.0012", "Milvus collection not found", 404)
+    RAG_EMBEDDING_FAILED: Final = ErrorCode("rag.0013", "Embedding generation failed", 500)
 
     # ========== 配比模块 ==========
     RATIO_TASK_NOT_FOUND: Final = ErrorCode("ratio.0001", "Ratio task not found", 404)

runtime/datamate-python/app/core/config.py

@@ -32,6 +32,8 @@
     ChunkUploadPreRequest
 )
 
+from .knowledge_gen import KnowledgeBase, RagFile
+
 __all__ = [
     "Dataset",
     "DatasetTag",
@@ -48,4 +50,6 @@
     "CategoryRelation",
     "OperatorRelease",
     "ChunkUploadPreRequest",
+    "KnowledgeBase",
+    "RagFile",
 ]

runtime/datamate-python/app/core/exception/codes.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, String, TIMESTAMP, Text, JSON
+from sqlalchemy import Column, String, TIMESTAMP, Text
 from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql import func