Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions frontend/src/pages/SynthesisTask/CreateTask.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ export default function SynthesisTaskCreate() {
| "PARAGRAPH_CHUNK"
| "FIXED_LENGTH_CHUNK"
| "CUSTOM_SEPARATOR_CHUNK",
chunkSize: 3000,
overlapSize: 100,
chunkSize: 500,
overlapSize: 50,
delimiter: "",
});

Expand Down
2 changes: 1 addition & 1 deletion frontend/src/pages/SynthesisTask/SynthFileTask.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ export default function SynthFileTask() {
danger: true,
confirm: {
title: t('synthesisTask.fileTask.confirm.deleteTitle'),
description: t('synthesisTask.fileTask.confirm.deleteDescription', { name: synthesisTask?.name || '' }),
description: t('synthesisTask.fileTask.confirm.deleteDescription', { name: taskInfo?.name || '' }),
okText: t('dataManagement.confirm.deleteConfirm'),
cancelText: t('dataManagement.confirm.deleteCancel'),
onConfirm: handleDelete,
Expand Down
65 changes: 50 additions & 15 deletions frontend/src/pages/SynthesisTask/components/SynthesisTaskTab.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ export default function SynthesisTaskTab() {
const [evalLoading, setEvalLoading] = useState(false);
const [models, setModels] = useState<ModelI[]>([]);
const [modelLoading, setModelLoading] = useState(false);
const [archiveModalVisible, setArchiveModalVisible] = useState(false);
const [currentArchiveTask, setCurrentArchiveTask] = useState<SynthesisTask | null>(null);
const [archiveFormat, setArchiveFormat] = useState<string>("alpaca");

const [evalForm] = Form.useForm();

Expand Down Expand Up @@ -210,7 +213,7 @@ export default function SynthesisTaskTab() {
title: t('synthesisTask.home.columns.actions'),
key: "actions",
fixed: "right" as const,
width: 120,
width: 160,
render: (_: unknown, task: SynthesisTask) => (
<div className="flex items-center justify-start gap-1">
<Tooltip title={t('synthesisTask.actions.viewDetail')}>
Expand All @@ -234,15 +237,7 @@ export default function SynthesisTaskTab() {
type="text"
className="hover:bg-green-50 p-1 h-7 w-7 flex items-center justify-center text-green-600"
icon={<FolderOpenOutlined />}
onClick={() => {
Modal.confirm({
title: t('synthesisTask.home.confirm.archiveTitle'),
content: t('synthesisTask.home.confirm.archiveContent', { name: task.name }),
okText: t('synthesisTask.actions.archive'),
cancelText: t('synthesisTask.actions.cancel'),
onOk: () => handleArchiveTask(task),
});
}}
onClick={() => openArchiveModal(task)}
/>
</Tooltip>
<Tooltip title={t('synthesisTask.actions.delete')}>
Expand Down Expand Up @@ -276,9 +271,8 @@ export default function SynthesisTaskTab() {
},
];

const handleArchiveTask = async (task: SynthesisTask) => {
const handleArchiveTask = async (task: SynthesisTask, format: string = "alpaca") => {
try {
// 1. 创建目标数据集(使用简单的默认命名 + 随机后缀,可后续扩展为弹窗自定义)
const randomSuffix = Math.random().toString(36).slice(2, 8);
const datasetReq: {
name: string;
Expand All @@ -302,18 +296,28 @@ export default function SynthesisTaskTab() {
return;
}

// 2. 调用后端归档接口,将合成数据写入该数据集
await archiveSynthesisTaskToDatasetUsingPost(task.id, datasetId);
await archiveSynthesisTaskToDatasetUsingPost(task.id, datasetId, format);

message.success(t('synthesisTask.home.archive.success'));
// 3. 可选:跳转到数据集详情页
navigate(`/data/management/detail/${datasetId}`);
} catch (e) {
console.error(e);
message.error(t('synthesisTask.home.archive.failed'));
}
};

const openArchiveModal = (task: SynthesisTask) => {
setCurrentArchiveTask(task);
setArchiveFormat("alpaca");
setArchiveModalVisible(true);
};

const handleArchiveConfirm = async () => {
if (!currentArchiveTask) return;
setArchiveModalVisible(false);
await handleArchiveTask(currentArchiveTask, archiveFormat);
};

const openEvalModal = (task: SynthesisTask) => {
setCurrentEvalTask(task);
setEvalModalVisible(true);
Expand Down Expand Up @@ -464,6 +468,37 @@ export default function SynthesisTaskTab() {
/>
</Card>

<Modal
title={t('synthesisTask.home.confirm.archiveTitle')}
open={archiveModalVisible}
onCancel={() => {
setArchiveModalVisible(false);
setCurrentArchiveTask(null);
}}
onOk={handleArchiveConfirm}
okText={t('synthesisTask.actions.archive')}
cancelText={t('synthesisTask.actions.cancel')}
>
<div>
<p>{t('synthesisTask.home.confirm.archiveContent', { name: currentArchiveTask?.name })}</p>
<div style={{ marginTop: 16 }}>
<span style={{ marginRight: 8 }}>导出格式:</span>
<Select
value={archiveFormat}
style={{ width: 120 }}
options={[
{ label: "Alpaca", value: "alpaca" },
{ label: "ShareGPT", value: "sharegpt" },
{ label: "原始格式", value: "raw" },
]}
onChange={(value) => {
setArchiveFormat(value);
}}
/>
</div>
</div>
</Modal>

<Modal
title={t('synthesisTask.home.modal.evalTitle')}
open={evalModalVisible}
Expand Down
31 changes: 14 additions & 17 deletions frontend/src/pages/SynthesisTask/synthesis-api.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { get, post, del } from "@/utils/request";
import { get, post, del, patch } from "@/utils/request";

// 创建数据合成任务
export function createSynthesisTaskUsingPost(data: Record<string, unknown>) {
Expand Down Expand Up @@ -65,28 +65,25 @@ export function getPromptByTypeUsingGet(synthType: string) {
}

// 将合成任务数据归档到已存在的数据集中
export function archiveSynthesisTaskToDatasetUsingPost(taskId: string, datasetId: string) {
return post(`/api/synthesis/gen/task/${taskId}/export-dataset/${datasetId}`);
export function archiveSynthesisTaskToDatasetUsingPost(
taskId: string,
datasetId: string,
format: string = "alpaca"
) {
return post(`/api/synthesis/gen/task/${taskId}/export-dataset/${datasetId}?format=${format}`);
}

// ---------------- 数据记录级别:chunk 与 synthesis data ----------------

// 根据 chunkId 删除单个 chunk 及其下所有合成数据
// 删除 chunk 及其关联的合成数据
export function deleteChunkWithDataUsingDelete(chunkId: string) {
return del(`/api/synthesis/gen/chunk/${chunkId}`);
}

// 删除某个 chunk 下的所有合成数据,返回删除条数
export function deleteSynthesisDataByChunkUsingDelete(chunkId: string) {
return del(`/api/synthesis/gen/chunk/${chunkId}/data`);
}

// 批量删除合成数据记录
export function batchDeleteSynthesisDataUsingDelete(body: { ids: string[] }) {
return del(`/api/synthesis/gen/data/batch`, null, { body: JSON.stringify(body) });
// 批量删除合成数据
export function batchDeleteSynthesisDataUsingDelete(data: { ids: string[] }) {
return del("/api/synthesis/gen/data/batch", data as unknown as Record<string, never>);
}

// 更新单条合成数据的完整 JSON 内容
export function updateSynthesisDataUsingPatch(dataId: string, body: { data: Record<string, unknown> }) {
return post(`/api/synthesis/gen/data/${dataId}`, body, { method: "PATCH" });
// 更新合成数据
export function updateSynthesisDataUsingPatch(dataId: string, data: { data: Record<string, unknown> }) {
return patch(`/api/synthesis/gen/data/${dataId}`, data as unknown as Record<string, never>);
}
23 changes: 23 additions & 0 deletions frontend/src/utils/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,28 @@ class Request {
return this.request(fullURL, config);
}

/**
* PATCH请求
* @param {string} url - 请求URL
* @param {object} data - 请求体数据
* @param {object} options - 额外的fetch选项,包括showLoading, onUploadProgress, onDownloadProgress
*/
async patch(url, data = null, options = {}) {
const config = {
method: "PATCH",
credentials: "include",
mode: "cors",
headers: {
...this.defaultHeaders,
...options.headers,
},
body: data ? JSON.stringify(data) : undefined,
...options,
};

return this.request(this.baseURL + url, config);
}

/**
* 从 Content-Disposition 头中解析文件名
*/
Expand Down Expand Up @@ -662,6 +684,7 @@ export const get = request.get.bind(request);
export const post = request.post.bind(request);
export const put = request.put.bind(request);
export const del = request.delete.bind(request);
export const patch = request.patch.bind(request);
export const download = request.download.bind(request);
export const upload = request.upload.bind(request);

Expand Down
4 changes: 2 additions & 2 deletions frontend/vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export default defineConfig({
host: "0.0.0.0",
proxy: (() => {
const pythonProxyConfig = {
target: "http://localhost:32033",
target: "http://localhost:18000",
changeOrigin: true,
secure: false,
configure: (proxy: { on: (event: string, handler: (arg: unknown) => void) => void }) => {
Expand All @@ -32,7 +32,7 @@ export default defineConfig({
};

const javaProxyConfig = {
target: "http://localhost:32033",
target: "http://localhost:8080",
changeOrigin: true,
secure: false,
configure: (proxy: { on: (event: string, handler: (arg: unknown) => void) => void }) => {
Expand Down
2 changes: 1 addition & 1 deletion runtime/datamate-python/app/core/exception/middleware.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ async def dispatch(self, request: Request, call_next):
except Exception as exc:
# 捕获所有未处理的异常
logger.error(
f"Unhandled exception occurred at {request.method} {request.url.path}", exc,
f"Unhandled exception occurred at {request.method} {request.url.path}",
exc_info=True
)
return self._error_response(
Expand Down
6 changes: 6 additions & 0 deletions runtime/datamate-python/app/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
set_collection_scheduler,
)
from app.module.shared.schedule import Scheduler
from app.module.generation.service.task_executor import init_executor, shutdown_executor

setup_logging()
logger = get_logger(__name__)
Expand Down Expand Up @@ -68,10 +69,15 @@ def mask_db_url(url: str) -> Literal[b""] | str:
set_collection_scheduler(collection_scheduler)
await load_scheduled_collection_tasks()

# Initialize generation task executor
init_executor(max_workers=10, max_concurrent_tasks=5)
logger.info("Generation task executor initialized")

yield

# @shutdown
collection_scheduler.shutdown()
shutdown_executor()
logger.info("DataMate Python Backend shutting down ...\n\n")


Expand Down
Loading
Loading