Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.datamate.common.domain.service.FileService;
import com.datamate.common.domain.utils.AnalyzerUtils;
import com.datamate.common.domain.utils.ArchiveAnalyzer;
import com.datamate.common.domain.utils.CommonUtils;
import com.datamate.common.infrastructure.exception.BusinessAssert;
import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.CommonErrorCode;
Expand Down Expand Up @@ -189,7 +190,7 @@ private DatasetFile getDatasetFile(Path path, Map<String, DatasetFile> datasetFi
} else {
DatasetFile exist = datasetFilesMap.get(path.toString());
if (exist == null) {
datasetFile.setId("file-" + datasetFile.getFileName());
datasetFile.setId(datasetFile.getFileName());
datasetFile.setFileSize(path.toFile().length());
} else {
datasetFile = exist;
Expand All @@ -202,12 +203,21 @@ private DatasetFile getDatasetFile(Path path, Map<String, DatasetFile> datasetFi
* 获取文件详情
*/
@Transactional(readOnly = true)
public DatasetFile getDatasetFile(String datasetId, String fileId) {
public DatasetFile getDatasetFile(Dataset dataset, String fileId) {
if (dataset != null && !CommonUtils.isUUID(fileId) && !fileId.startsWith(".")) {
DatasetFile file = new DatasetFile();
file.setId(fileId);
file.setFileName(fileId);
file.setDatasetId(dataset.getId());
file.setFileSize(0L);
file.setFilePath(dataset.getPath() + File.separator + fileId);
return file;
}
DatasetFile file = datasetFileRepository.getById(fileId);
if (file == null) {
if (file == null || dataset == null) {
throw new IllegalArgumentException("File not found: " + fileId);
}
if (!file.getDatasetId().equals(datasetId)) {
if (!file.getDatasetId().equals(dataset.getId())) {
throw new IllegalArgumentException("File does not belong to the specified dataset");
}
return file;
Expand All @@ -218,11 +228,13 @@ public DatasetFile getDatasetFile(String datasetId, String fileId) {
*/
@Transactional
public void deleteDatasetFile(String datasetId, String fileId) {
DatasetFile file = getDatasetFile(datasetId, fileId);
Dataset dataset = datasetRepository.getById(datasetId);
DatasetFile file = getDatasetFile(dataset, fileId);
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
datasetFileRepository.removeById(fileId);
dataset.removeFile(file);
if (CommonUtils.isUUID(fileId)) {
dataset.removeFile(file);
}
datasetRepository.updateById(dataset);
// 删除文件时,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录
if (file.getFilePath().startsWith(dataset.getPath())) {
Expand All @@ -239,10 +251,10 @@ public void deleteDatasetFile(String datasetId, String fileId) {
* 下载文件
*/
@Transactional(readOnly = true)
public Resource downloadFile(String datasetId, String fileId) {
DatasetFile file = getDatasetFile(datasetId, fileId);
public Resource downloadFile(DatasetFile file) {
try {
Path filePath = Paths.get(file.getFilePath()).normalize();
log.info("start download file {}", file.getFilePath());
Resource resource = new UrlResource(filePath.toUri());
if (resource.exists()) {
return resource;
Expand Down Expand Up @@ -638,12 +650,12 @@ public void deleteDirectory(String datasetId, String prefix) {
*/
@Transactional
public void renameFile(String datasetId, String fileId, RenameFileRequest request) {
DatasetFile file = getDatasetFile(datasetId, fileId);
Dataset dataset = datasetRepository.getById(datasetId);
if (dataset == null) {
throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
}

DatasetFile file = getDatasetFile(dataset, fileId);
String newName = Optional.ofNullable(request.getNewName()).orElse("").trim();
if (newName.isEmpty()) {
throw BusinessException.of(CommonErrorCode.PARAM_ERROR);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.datamate.common.infrastructure.config.PgJsonTypeHandler;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.*;
Expand Down Expand Up @@ -32,7 +33,7 @@ public class DatasetFile {
private Long fileSize; // bytes
private String checkSum;
private String tags;
private LocalDateTime tagsUpdatedAt;
@TableField(typeHandler = PgJsonTypeHandler.class)
private String metadata;
private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR
private LocalDateTime uploadTime;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.common.interfaces.PagingQuery;
import com.datamate.datamanagement.application.DatasetApplicationService;
import com.datamate.datamanagement.application.DatasetFileApplicationService;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
Expand All @@ -18,6 +20,7 @@
import com.datamate.datamanagement.interfaces.dto.RenameDirectoryRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
Expand All @@ -34,15 +37,13 @@
*/
@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping("/data-management/datasets/{datasetId}/files")
public class DatasetFileController {

private final DatasetFileApplicationService datasetFileApplicationService;

@Autowired
public DatasetFileController(DatasetFileApplicationService datasetFileApplicationService) {
this.datasetFileApplicationService = datasetFileApplicationService;
}
private final DatasetApplicationService datasetApplicationService;

@GetMapping
public Response<PagedResponse<DatasetFile>> getDatasetFiles(
Expand All @@ -66,7 +67,8 @@ public ResponseEntity<Response<DatasetFileResponse>> getDatasetFileById(
@PathVariable("datasetId") String datasetId,
@PathVariable("fileId") String fileId) {
try {
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
Dataset dataset = datasetApplicationService.getDataset(datasetId);
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId);
return ResponseEntity.ok(Response.ok(DatasetConverter.INSTANCE.convertToResponse(datasetFile)));
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
Expand All @@ -90,17 +92,20 @@ public ResponseEntity<Response<Void>> deleteDatasetFile(
public ResponseEntity<Resource> downloadDatasetFileById(@PathVariable("datasetId") String datasetId,
@PathVariable("fileId") String fileId) {
try {
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
Resource resource = datasetFileApplicationService.downloadFile(datasetId, fileId);
Dataset dataset = datasetApplicationService.getDataset(datasetId);
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId);
Resource resource = datasetFileApplicationService.downloadFile(datasetFile);

return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + datasetFile.getFileName() + "\"")
.body(resource);
} catch (IllegalArgumentException e) {
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
} catch (Exception e) {
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.datamate.common.domain.utils;

import java.io.File;
import java.util.UUID;

/**
* 通用工具类
Expand All @@ -21,4 +22,20 @@ public static String trimFilePath(String filePath) {
}
return filename;
}

/**
* 判断字符串是否是uuid
*
* @param str 要判断的字符串
* @return 判断结果
*/
public static boolean isUUID(String str) {
if (str == null) return false;
try {
UUID.fromString(str);
return true;
} catch (IllegalArgumentException e) {
return false;
}
}
}
58 changes: 2 additions & 56 deletions frontend/src/pages/DataCollection/collection.apis.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { get, post, put, del } from "@/utils/request";
import {get, post, put, del, download} from "@/utils/request";

// 数据源任务相关接口
export function queryTasksUsingGet(params?: any) {
Expand All @@ -9,21 +9,6 @@ export function createTaskUsingPost(data: any) {
return post("/api/data-collection/tasks", data);
}

export function queryTaskByIdUsingGet(id: string | number) {
return get(`/api/data-collection/tasks/${id}`);
}

export function updateTaskByIdUsingPut(
id: string | number,
data: any
) {
return put(`/api/data-collection/tasks/${id}`, data);
}

export function queryTaskDetailsByIdUsingGet(id: string | number) {
return get(`/api/data-collection/tasks/${id}`);
}

export function queryDataXTemplatesUsingGet(params?: any) {
return get("/api/data-collection/templates", params);
}
Expand All @@ -50,45 +35,6 @@ export function queryExecutionLogUsingPost(params?: any) {
return get("/api/data-collection/executions", params);
}

export function queryExecutionLogByIdUsingGet(id: string | number) {
return get(`/api/data-collection/executions/${id}`);
}

export function queryExecutionLogContentByIdUsingGet(id: string | number) {
return get(`/api/data-collection/executions/${id}/log`);
}

export async function queryExecutionLogFileByIdUsingGet(id: string | number) {
const token = localStorage.getItem("token") || sessionStorage.getItem("token");
const resp = await fetch(`/api/data-collection/executions/${id}/log`, {
method: "GET",
headers: {
...(token ? { Authorization: `Bearer ${token}` } : {}),
},
credentials: "include",
});

if (!resp.ok) {
let detail = "";
try {
detail = await resp.text();
} catch {
detail = resp.statusText;
}
const err: any = new Error(detail || `HTTP error ${resp.status}`);
err.status = resp.status;
err.data = detail;
throw err;
}

const contentDisposition = resp.headers.get("content-disposition") || "";
const filenameMatch = contentDisposition.match(/filename\*?=(?:UTF-8''|\")?([^;\"\n]+)/i);
const filename = filenameMatch?.[1] ? decodeURIComponent(filenameMatch[1].replace(/\"/g, "").trim()) : `execution_${id}.log`;
const blob = await resp.blob();
return { blob, filename };
}

// 监控统计相关接口
export function queryCollectionStatisticsUsingGet(params?: any) {
return get("/api/data-collection/monitor/statistics", params);
return await download(`/api/data-collection/executions/${id}/log`, null, undefined, "preview");
}
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ async def get_execution_log(

filename = path.name
headers = {
"Content-Disposition": f'inline; filename="{filename}"'
"Content-Disposition": f'inline; filename={filename}'
}
return FileResponse(
path=str(path),
Expand Down
Loading