diff --git a/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/SubGraphRecord.java b/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/SubGraphRecord.java index c25ed21ce..5aac5fa90 100644 --- a/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/SubGraphRecord.java +++ b/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/SubGraphRecord.java @@ -22,8 +22,10 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Setter; @Getter +@Setter @AllArgsConstructor @EqualsAndHashCode(callSuper = false) public class SubGraphRecord extends BaseRecord { diff --git a/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java b/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java index 73963cb92..955b5a578 100644 --- a/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java +++ b/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/physical/sink/impl/Neo4jSinkWriter.java @@ -29,19 +29,14 @@ import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGPropertyRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName; +import com.antgroup.openspg.cloudext.interfaces.graphstore.util.TypeChecker; import com.antgroup.openspg.server.common.model.project.Project; import com.google.common.collect.Lists; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -188,7 +183,11 @@ private void writeNode(SubGraphRecord.Node node) { List vertexRecords = Lists.newArrayList(); List properties = Lists.newArrayList(); for (Map.Entry entry : node.getProperties().entrySet()) { - properties.add(new LPGPropertyRecord(entry.getKey(), entry.getValue())); + Object entryValue = entry.getValue(); + if (!TypeChecker.isArrayOrCollectionOfPrimitives(entryValue)) { + entryValue = JSON.toJSONString(entryValue); + } + properties.add(new LPGPropertyRecord(entry.getKey(), entryValue)); } VertexRecord vertexRecord = new VertexRecord(node.getId(), label, properties); vertexRecords.add(vertexRecord); @@ -226,7 +225,11 @@ private void writeEdge(SubGraphRecord.Edge edge) { List edgeRecords = Lists.newArrayList(); List properties = Lists.newArrayList(); for (Map.Entry entry : edge.getProperties().entrySet()) { - properties.add(new LPGPropertyRecord(entry.getKey(), entry.getValue())); + Object entryValue = entry.getValue(); + if (!TypeChecker.isArrayOrCollectionOfPrimitives(entryValue)) { + entryValue = JSON.toJSONString(entryValue); + } + properties.add(new LPGPropertyRecord(entry.getKey(), entryValue)); } EdgeTypeName edgeTypeName = diff --git a/cloudext/impl/graph-store/tugraph/src/main/java/com/antgroup/openspg/cloudext/impl/graphstore/tugraph/TuGraphStoreClient.java b/cloudext/impl/graph-store/tugraph/src/main/java/com/antgroup/openspg/cloudext/impl/graphstore/tugraph/TuGraphStoreClient.java index 27528bd0c..ddcfb9bce 100644 --- a/cloudext/impl/graph-store/tugraph/src/main/java/com/antgroup/openspg/cloudext/impl/graphstore/tugraph/TuGraphStoreClient.java +++ b/cloudext/impl/graph-store/tugraph/src/main/java/com/antgroup/openspg/cloudext/impl/graphstore/tugraph/TuGraphStoreClient.java @@ -48,7 +48,6 @@ import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.operation.SchemaAtomicOperationEnum; import com.antgroup.openspg.cloudext.interfaces.graphstore.util.TypeNameUtils; import com.antgroup.openspg.core.schema.model.type.BasicTypeEnum; -import com.antgroup.openspg.server.api.facade.ApiConstants; import com.antgroup.tugraph.TuGraphDbRpcClient; import com.google.common.collect.Lists; import java.io.InputStream; @@ -78,13 +77,16 @@ public class TuGraphStoreClient extends BaseLPGGraphStoreClient { @Getter private final LPGTypeNameConvertor typeNameConvertor; @Getter private final String connUrl; + private static final String ACCESS_ID = "accessId"; + private static final String ACCESS_KEY = "accessKey"; + private static final String TIMEOUT = "timeout"; + public TuGraphStoreClient(String connUrl, LPGTypeNameConvertor typeNameConvertor) { UriComponents uriComponents = UriComponentsBuilder.fromUriString(connUrl).build(); this.connUrl = connUrl; this.graphName = uriComponents.getQueryParams().getFirst(TuGraphConstants.GRAPH_NAME); this.timeout = - Double.parseDouble( - String.valueOf(uriComponents.getQueryParams().getFirst(ApiConstants.TIMEOUT))); + Double.parseDouble(String.valueOf(uriComponents.getQueryParams().getFirst(TIMEOUT))); this.client = initTuGraphClient(uriComponents); this.internalIdGenerator = new NoChangedIdGenerator(); this.typeNameConvertor = typeNameConvertor; @@ -149,8 +151,8 @@ public void close() throws Exception { private TuGraphDbRpcClient initTuGraphClient(UriComponents uriComponents) { String host = String.format("%s:%s", uriComponents.getHost(), uriComponents.getPort()); - String accessId = uriComponents.getQueryParams().getFirst(ApiConstants.ACCESS_ID); - String accessKey = uriComponents.getQueryParams().getFirst(ApiConstants.ACCESS_KEY); + String accessId = uriComponents.getQueryParams().getFirst(ACCESS_ID); + String accessKey = uriComponents.getQueryParams().getFirst(ACCESS_KEY); TuGraphDbRpcClient client = null; try { client = new TuGraphDbRpcClient(host, accessId, accessKey); diff --git a/cloudext/impl/object-storage/minio/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/minio/MinioClient.java b/cloudext/impl/object-storage/minio/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/minio/MinioClient.java index ddb3bc7e9..962fe955e 100644 --- a/cloudext/impl/object-storage/minio/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/minio/MinioClient.java +++ b/cloudext/impl/object-storage/minio/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/minio/MinioClient.java @@ -37,7 +37,9 @@ import java.io.OutputStream; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.utils.IOUtils; @@ -72,6 +74,7 @@ private io.minio.MinioClient initMinioClient(UriComponents uriComponents) { @Override public Boolean saveData(String bucketName, byte[] data, String fileKey) { + Long start = System.currentTimeMillis(); ByteArrayInputStream inputStream = null; try { inputStream = new ByteArrayInputStream(data); @@ -87,12 +90,14 @@ public Boolean saveData(String bucketName, byte[] data, String fileKey) { log.error(e.getMessage(), e); throw new RuntimeException("minio saveData Exception:" + e.getMessage(), e); } finally { + log.info("minio saveData cons:{} fileKey:{}", System.currentTimeMillis() - start, fileKey); IOUtils.closeQuietly(inputStream); } } @Override public byte[] getData(String bucketName, String fileKey) { + Long start = System.currentTimeMillis(); try { GetObjectArgs getObjectArgs = GetObjectArgs.builder().bucket(bucketName).object(fileKey).build(); @@ -100,6 +105,8 @@ public byte[] getData(String bucketName, String fileKey) { } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException("minio getData Exception:" + e.getMessage(), e); + } finally { + log.info("minio getData cons:{} fileKey:{}", System.currentTimeMillis() - start, fileKey); } } @@ -239,10 +246,14 @@ public String getUrlWithoutExpiration(String bucketName, String fileKey) { @Override public Boolean removeObject(String bucketName, String fileKey) { try { - RemoveObjectArgs removeObjectArgs = - RemoveObjectArgs.builder().bucket(bucketName).object(fileKey).build(); - minioClient.removeObject(removeObjectArgs); - return true; + if (isDirectory(bucketName, fileKey)) { + return removeDirectory(bucketName, fileKey); + } else { + RemoveObjectArgs removeObjectArgs = + RemoveObjectArgs.builder().bucket(bucketName).object(fileKey).build(); + minioClient.removeObject(removeObjectArgs); + return true; + } } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException("minio removeObject Exception:" + e.getMessage(), e); @@ -252,26 +263,64 @@ public Boolean removeObject(String bucketName, String fileKey) { @Override public Boolean removeDirectory(String bucketName, String directoryPath) { try { - Iterable> objects = + List files = getAllFilesRecursively(bucketName, directoryPath); + + for (String file : files) { + log.info("minio Deleting: " + file); + if (file.startsWith(directoryPath)) { + removeObject(bucketName, file); + } + } + return true; + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new RuntimeException("minio removeDirectory Exception:" + e.getMessage(), e); + } + } + + @Override + public Boolean isDirectory(String bucketName, String path) { + try { + Iterable> items = + minioClient.listObjects( + ListObjectsArgs.builder().bucket(bucketName).prefix(path).recursive(false).build()); + for (Result result : items) { + Item item = result.get(); + if (item.isDir() || !item.objectName().equals(path)) { + return true; + } + } + return false; + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new RuntimeException("minio isDirectory Exception:" + e.getMessage(), e); + } + } + + @Override + public List getAllFilesRecursively(String bucketName, String directoryPath) { + List filePaths = new ArrayList<>(); + try { + Iterable> items = minioClient.listObjects( ListObjectsArgs.builder() .bucket(bucketName) .prefix(directoryPath) - .recursive(true) + .recursive(false) .build()); - - for (Result result : objects) { + for (Result result : items) { Item item = result.get(); - log.info("minio Deleting: " + item.objectName()); - if (item.objectName().startsWith(directoryPath)) { - removeObject(bucketName, item.objectName()); + if (item.isDir()) { + filePaths.addAll(getAllFilesRecursively(bucketName, item.objectName())); + } else { + filePaths.add(item.objectName()); } } - return true; } catch (Exception e) { log.error(e.getMessage(), e); - throw new RuntimeException("minio removeDirectory Exception:" + e.getMessage(), e); + throw new RuntimeException("minio getAllFilesRecursively Exception: " + e.getMessage(), e); } + return filePaths; } @Override @@ -287,6 +336,27 @@ public Long getContentLength(String bucketName, String objectName) { } } + @Override + public long getStorageSize(String bucketName, String path) { + try { + Iterable> items = + minioClient.listObjects( + ListObjectsArgs.builder().bucket(bucketName).prefix(path).recursive(true).build()); + + long totalSizeInBytes = 0; + for (Result itemResult : items) { + Item item = itemResult.get(); + if (!item.isDir()) { + totalSizeInBytes += item.size(); + } + } + return totalSizeInBytes; + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new RuntimeException("minio getStorageSize Exception:" + e.getMessage(), e); + } + } + public void makeBucket(String bucketName) { try { boolean isExist = diff --git a/cloudext/impl/object-storage/oss/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/oss/OSSClient.java b/cloudext/impl/object-storage/oss/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/oss/OSSClient.java index f3f905f45..5cc68a161 100644 --- a/cloudext/impl/object-storage/oss/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/oss/OSSClient.java +++ b/cloudext/impl/object-storage/oss/src/main/java/com/antgroup/openspg/cloudext/impl/objectstorage/oss/OSSClient.java @@ -33,7 +33,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.web.util.UriComponents; @@ -42,6 +44,8 @@ @Slf4j public class OSSClient implements ObjectStorageClient { + private static final String SLASH = "/"; + private final OSS ossClient; @Getter private final String connUrl; @@ -209,8 +213,12 @@ public String getUrlWithoutExpiration(String bucketName, String fileKey) { @Override public Boolean removeObject(String bucketName, String fileKey) { try { - ossClient.deleteObject(bucketName, fileKey); - return true; + if (isDirectory(bucketName, fileKey)) { + return removeDirectory(bucketName, fileKey); + } else { + ossClient.deleteObject(bucketName, fileKey); + return true; + } } catch (Exception e) { log.error(e.getMessage(), e); throw new RuntimeException("OSS removeObject Exception:" + e.getMessage(), e); @@ -220,25 +228,67 @@ public Boolean removeObject(String bucketName, String fileKey) { @Override public Boolean removeDirectory(String bucketName, String directoryPath) { try { - ObjectListing objectListing; + List files = getAllFilesRecursively(bucketName, directoryPath); + + for (String file : files) { + log.info("OSS Deleting: " + file); + if (file.startsWith(directoryPath)) { + removeObject(bucketName, file); + } + } + return true; + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new RuntimeException("OSS removeDirectory Exception: " + e.getMessage(), e); + } + } + @Override + public Boolean isDirectory(String bucketName, String path) { + try { + ObjectListing objectListing = ossClient.listObjects(bucketName, path); + if (!objectListing.getCommonPrefixes().isEmpty() + || !objectListing.getObjectSummaries().isEmpty()) { + if (path.endsWith(SLASH)) { + return true; + } + } + for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) { + String objectName = objectSummary.getKey(); + if (objectName.endsWith(SLASH) || !objectName.equals(path)) { + return true; + } + } + return false; + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new RuntimeException("OSS isDirectory Exception: " + e.getMessage(), e); + } + } + + @Override + public List getAllFilesRecursively(String bucketName, String directoryPath) { + List filePaths = new ArrayList<>(); + try { + String nextMarker; do { - objectListing = ossClient.listObjects(bucketName, directoryPath); - for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) { - String objectName = objectSummary.getKey(); - log.info("OSS Deleting: " + objectName); - if (objectName.startsWith(directoryPath)) { - removeObject(bucketName, objectName); + ObjectListing objectListing = ossClient.listObjects(bucketName, directoryPath); + List objectSummaries = objectListing.getObjectSummaries(); + for (OSSObjectSummary summary : objectSummaries) { + if (summary.getKey().endsWith(SLASH)) { + filePaths.addAll(getAllFilesRecursively(bucketName, summary.getKey())); + } else { + filePaths.add(summary.getKey()); } } - objectListing.getNextMarker(); - } while (objectListing.isTruncated()); + nextMarker = objectListing.getNextMarker(); + } while (nextMarker != null); - return true; } catch (Exception e) { - log.error(e.getMessage(), e); - throw new RuntimeException("OSS removeDirectory Exception:" + e.getMessage(), e); + log.error("OSS getAllFilesRecursively Exception: " + e.getMessage(), e); + throw new RuntimeException("OSS getAllFilesRecursively Exception: " + e.getMessage(), e); } + return filePaths; } @Override @@ -251,4 +301,25 @@ public Long getContentLength(String bucketName, String objectName) { throw new RuntimeException("OSS getContentLength Exception:" + e.getMessage(), e); } } + + @Override + public long getStorageSize(String bucketName, String path) { + long totalSizeInBytes = 0L; + + try { + String nextMarker; + do { + ObjectListing objectListing = ossClient.listObjects(bucketName, path); + for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) { + totalSizeInBytes += objectSummary.getSize(); + } + nextMarker = objectListing.getNextMarker(); + } while (nextMarker != null); + return totalSizeInBytes; + + } catch (Exception e) { + log.error(e.getMessage(), e); + throw new RuntimeException("OSS getStorageSize Exception:" + e.getMessage(), e); + } + } } diff --git a/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java b/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java index 9f82f121a..f3ac53284 100644 --- a/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java +++ b/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java @@ -31,8 +31,8 @@ import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.MatchQuery; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.QueryGroup; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.TermQuery; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import com.antgroup.openspg.common.util.StringUtils; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.dtflys.forest.http.ForestResponse; import com.google.gson.reflect.TypeToken; import java.net.URLEncoder; diff --git a/cloudext/impl/search-engine/neo4j/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/neo4j/Neo4jSearchClient.java b/cloudext/impl/search-engine/neo4j/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/neo4j/Neo4jSearchClient.java index 5191d1132..f1c2c5cb1 100644 --- a/cloudext/impl/search-engine/neo4j/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/neo4j/Neo4jSearchClient.java +++ b/cloudext/impl/search-engine/neo4j/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/neo4j/Neo4jSearchClient.java @@ -22,6 +22,7 @@ import com.antgroup.openspg.cloudext.interfaces.searchengine.model.idx.schema.IdxSchema; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.SearchRequest; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.BaseQuery; +import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.CustomSearchQuery; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.FullTextSearchQuery; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.VectorSearchQuery; import com.antgroup.openspg.common.util.neo4j.Neo4jCommonUtils; @@ -106,9 +107,12 @@ private List doNeo4jSearch(SearchRequest request) { VectorSearchQuery q = (VectorSearchQuery) query; return client.vectorSearch( q.getLabel(), q.getPropertyKey(), q.getQueryVector(), topk, indexName, q.getEfSearch()); + } else if (query instanceof CustomSearchQuery) { + CustomSearchQuery q = (CustomSearchQuery) query; + return client.customSearch(q.getCustomQuery()); } else { throw new RuntimeException( - "Neo4jSearchClient only supports FullTextSearchQuery and VectorSearchQuery."); + "Neo4jSearchClient only supports FullTextSearchQuery, VectorSearchQuery and CustomSearchQuery."); } } diff --git a/cloudext/interface/computing-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/computingengine/ComputingEngineConstants.java b/cloudext/interface/computing-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/computingengine/ComputingEngineConstants.java index 54a19226c..01a387d30 100644 --- a/cloudext/interface/computing-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/computingengine/ComputingEngineConstants.java +++ b/cloudext/interface/computing-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/computingengine/ComputingEngineConstants.java @@ -15,4 +15,5 @@ public class ComputingEngineConstants { public static final String USER_NUMBER = "userNumber"; + public static final String DEFAULT_NUMBER = "164072"; } diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/LPGDataDefinitionService.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/LPGDataDefinitionService.java index 32c25079f..d99c894fc 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/LPGDataDefinitionService.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/LPGDataDefinitionService.java @@ -37,6 +37,7 @@ public interface LPGDataDefinitionService { */ LPGSchema querySchema(); + @Deprecated List queryAllVertexLabels(); /** diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java index f4284d4d8..bc58c161e 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java @@ -71,6 +71,7 @@ public static BaseAdvancedRecord toAdvancedRecord( BaseAdvancedRecord advancedRecord = null; switch (baseSpgType.getSpgTypeEnum()) { case ENTITY_TYPE: + case INDEX_TYPE: advancedRecord = new EntityRecord( (EntityType) baseSpgType, diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/cmd/PageRankCompete.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/cmd/PageRankCompete.java index 4f061d420..d92f4e529 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/cmd/PageRankCompete.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/cmd/PageRankCompete.java @@ -23,6 +23,10 @@ public class PageRankCompete { private String targetVertexType; private Float dampingFactor = 0.85f; private Integer maxIterations = 20; + private Integer parallel = 16; + private Boolean directed = false; + private Float tolerance = 0.00001f; + private Integer topK = 50; public PageRankCompete(List startVertices, String targetVertexType) { this.startVertices = startVertices; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java index 43fe5cc0a..f351f7a5d 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java @@ -14,7 +14,7 @@ package com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.VertexType; -import com.antgroup.openspg.server.api.facade.ApiConstants; +import com.google.common.collect.Lists; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,12 +35,18 @@ public class VertexRecord extends BaseLPGRecord { private final String id; + private static final long DEFAULT_VERSION = 0L; + public VertexRecord(String id, String vertexType, List properties) { super(LPGRecordTypeEnum.VERTEX, properties); this.id = id; this.vertexType = vertexType; } + public VertexRecord(String id, String vertexType) { + this(id, vertexType, Lists.newArrayList()); + } + @Override public Map toPropertyMapWithId() { Map otherProperties = toPropertyMap(); @@ -55,7 +61,7 @@ public Map> toPropertyMapWithIdAndVersion() { otherProperties.forEach( (key, value) -> { TreeMap propertyVersion = new TreeMap<>(); - propertyVersion.put(ApiConstants.DEFAULT_VERSION, value); + propertyVersion.put(DEFAULT_VERSION, value); results.put(key, propertyVersion); }); return results; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/schema/EdgeTypeName.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/schema/EdgeTypeName.java index 8457e3e11..819647561 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/schema/EdgeTypeName.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/schema/EdgeTypeName.java @@ -17,6 +17,7 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; /** * {@link EdgeTypeName EdgeTypeName} is the unique identifier of {@link EdgeType EdgeType}, and it @@ -25,15 +26,16 @@ */ @Getter @AllArgsConstructor +@NoArgsConstructor @EqualsAndHashCode(callSuper = false) public class EdgeTypeName extends BaseValObj { - private final String startVertexType; + private String startVertexType; /** The label of edge type */ - private final String edgeLabel; + private String edgeLabel; - private final String endVertexType; + private String endVertexType; public static EdgeTypeName parse(String edgeTypeName) { String[] splits = edgeTypeName.split("_"); diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/util/TypeChecker.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/util/TypeChecker.java new file mode 100644 index 000000000..0bde51d81 --- /dev/null +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/util/TypeChecker.java @@ -0,0 +1,69 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.cloudext.interfaces.graphstore.util; + +import java.util.Collection; + +public class TypeChecker { + public static boolean isPrimitiveOrWrapper(Object obj) { + if (obj == null) { + return false; + } + Class clazz = obj.getClass(); + return clazz.isPrimitive() + || clazz.equals(Integer.class) + || clazz.equals(Double.class) + || clazz.equals(Float.class) + || clazz.equals(Long.class) + || clazz.equals(Short.class) + || clazz.equals(Byte.class) + || clazz.equals(Boolean.class) + || clazz.equals(Character.class); + } + + public static boolean isArrayOrCollectionOfPrimitives(Object obj) { + if (obj == null) { + return false; + } + if (obj instanceof Object[]) { + for (Object element : (Object[]) obj) { + if (!isPrimitiveOrWrapper(element)) { + return false; + } + } + return true; + } else if (obj instanceof Collection) { + for (Object element : (Collection) obj) { + if (!isPrimitiveOrWrapper(element)) { + return false; + } + } + return true; + } + return false; + } +} diff --git a/cloudext/interface/object-storage/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstorage/ObjectStorageClient.java b/cloudext/interface/object-storage/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstorage/ObjectStorageClient.java index cab28e5fb..74839c1f5 100644 --- a/cloudext/interface/object-storage/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstorage/ObjectStorageClient.java +++ b/cloudext/interface/object-storage/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstorage/ObjectStorageClient.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.InputStream; import java.util.Date; +import java.util.List; public interface ObjectStorageClient extends CloudExtClient { @@ -44,5 +45,11 @@ public interface ObjectStorageClient extends CloudExtClient { Boolean removeDirectory(String bucketName, String directoryPath); + Boolean isDirectory(String bucketName, String path); + + List getAllFilesRecursively(String bucketName, String directoryPath); + Long getContentLength(String bucketName, String objectName); + + long getStorageSize(String bucketName, String path); } diff --git a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/idx/record/IdxRecord.java b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/idx/record/IdxRecord.java index bd2d9d675..375da9e27 100644 --- a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/idx/record/IdxRecord.java +++ b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/idx/record/IdxRecord.java @@ -25,9 +25,18 @@ public class IdxRecord extends BaseValObj { @Setter private String idxName; + @Setter private String label; + private final String docId; private final double score; private final Map fields; + + public IdxRecord(String idxName, String docId, double score, Map fields) { + this.idxName = idxName; + this.docId = docId; + this.score = score; + this.fields = fields; + } } diff --git a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/CustomSearchQuery.java b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/CustomSearchQuery.java new file mode 100644 index 000000000..184ad0399 --- /dev/null +++ b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/CustomSearchQuery.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query; + +import lombok.Getter; +import lombok.NonNull; + +public class CustomSearchQuery extends BaseQuery { + + @Getter private final String customQuery; + + public CustomSearchQuery(@NonNull String customQuery) { + this.customQuery = customQuery; + } +} diff --git a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/FullTextSearchQuery.java b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/FullTextSearchQuery.java index 01ba1f34f..1be1ee2e0 100644 --- a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/FullTextSearchQuery.java +++ b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/FullTextSearchQuery.java @@ -22,13 +22,22 @@ public class FullTextSearchQuery extends BaseQuery { @Getter private final String queryString; @Getter private final List labelConstraints; + @Getter private final String propertyKey; public FullTextSearchQuery(@NonNull String queryString) { this(queryString, null); } public FullTextSearchQuery(@NonNull String queryString, @Nullable List labelConstraints) { + this(queryString, labelConstraints, null); + } + + public FullTextSearchQuery( + @NonNull String queryString, + @Nullable List labelConstraints, + @Nullable String propertyKey) { this.queryString = queryString; this.labelConstraints = labelConstraints; + this.propertyKey = propertyKey; } } diff --git a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/VectorSearchQuery.java b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/VectorSearchQuery.java index a6a5212b2..9c6b7616e 100644 --- a/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/VectorSearchQuery.java +++ b/cloudext/interface/search-engine/src/main/java/com/antgroup/openspg/cloudext/interfaces/searchengine/model/request/query/VectorSearchQuery.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query; +import javax.annotation.Nullable; import lombok.Getter; import lombok.NonNull; @@ -24,12 +25,12 @@ public class VectorSearchQuery extends BaseQuery { @Getter private final int efSearch; public VectorSearchQuery( - @NonNull String label, @NonNull String propertyKey, float @NonNull [] queryVector) { + @Nullable String label, @NonNull String propertyKey, float @NonNull [] queryVector) { this(label, propertyKey, queryVector, -1); } public VectorSearchQuery( - @NonNull String label, + @Nullable String label, @NonNull String propertyKey, float @NonNull [] queryVector, int efSearch) { diff --git a/common/util/pom.xml b/common/util/pom.xml index ee056e1fc..36c8d4408 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -25,7 +25,7 @@ com.antgroup.openspg.server - api-facade + core-schema-model commons-codec @@ -75,5 +75,13 @@ commons-io commons-io + + com.google.code.gson + gson + + + org.danilopianini + gson-extras + diff --git a/common/util/src/main/java/com/antgroup/openspg/common/constants/BuilderConstant.java b/common/util/src/main/java/com/antgroup/openspg/common/constants/BuilderConstant.java index 4a6781c56..181143412 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/constants/BuilderConstant.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/constants/BuilderConstant.java @@ -50,12 +50,14 @@ public class BuilderConstant { public static final String BUILDER_TYPE = "builderType"; public static final String KAG = "kag"; public static final String LOCAL = "local"; + public static final String AISTUDIO = "aistudio"; public static final String DISTRIBUTED = "distributed"; public static final String COMMAND = "command"; public static final String SPG_DEFAULT_COMMAND = - ". /openspg_venv/bin/activate " - + "&&/openspg_venv/bin/pip3.8 install openspg-kag==0.6.2b1 -i https://artifacts.antgroup-inc.cn/artifact/repositories/simple-dev/ " - + "&&python -c 'from kag.bridge.spg_server_bridge import SPGServerBridge; builder = SPGServerBridge(); builder.run_builder(%s,%s)'"; + "pip install openspg-kag-ant==0.7.0.202505062 -i https://artifacts.antgroup-inc.cn/artifact/repositories/simple-dev/ " + + "&&pip uninstall -y openspg-kag " + + "&&pip install openspg-kag==0.7.1.20250605 -i https://artifacts.antgroup-inc.cn/artifact/repositories/simple-dev/ " + + "&&python -c 'import kag_ant; from kag.bridge.spg_server_bridge import init_kag_config; init_kag_config(\"%s\",\"%s\"); from kag.bridge.spg_server_bridge import SPGServerBridge; builder = SPGServerBridge(); builder.run_builder(%s,\"%s\")'"; public static final String STRUCTURE = "structure"; public static final String STRUCTURED_BUILDER_CHAIN = "structured_builder_chain"; public static final String UNSTRUCTURED_BUILDER_CHAIN = "unstructured_builder_chain"; @@ -166,4 +168,9 @@ public class BuilderConstant { public static final String DELETE = "delete"; public static final String BATCH_VECTORIZER = "batch_vectorizer"; public static final String KAG_POST_PROCESSOR = "kag_post_processor"; + + public static final String TOKENS_COST = "tokensCost"; + public static final String TIME_COST = "timeCost"; + public static final String STORAGE_COST = "storageCost"; + public static final String COMPLETION_TOKENS = "completion_tokens"; } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/constants/SpgAppConstant.java b/common/util/src/main/java/com/antgroup/openspg/common/constants/SpgAppConstant.java index d9caf422b..d376ee6c3 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/constants/SpgAppConstant.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/constants/SpgAppConstant.java @@ -12,9 +12,12 @@ */ package com.antgroup.openspg.common.constants; +import com.google.common.collect.Lists; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.regex.Pattern; public class SpgAppConstant { @@ -57,6 +60,9 @@ public class SpgAppConstant { /** kag config */ public static final String KAG_CONFIG = "KAG_CONFIG"; + /** kag env */ + public static final String KAG_ENV = "KAG_ENV"; + /** kag config */ public static final String APPLICATION_PROPERTIES = "APPLICATION_PROPERTIES"; @@ -111,6 +117,9 @@ public class SpgAppConstant { /** currentLanguage */ public static final String CURRENT_LANGUAGE = "currentLanguage"; + /** LANGUAGE */ + public static final String LANGUAGE = "language"; + /** temperature */ public static final String TEMPERATURE = "temperature"; @@ -134,7 +143,17 @@ public class SpgAppConstant { /** hidden property */ public static Set HIDDEN_PROPERTY = - new HashSet(Arrays.asList("_content_vector", "_name_vector", "_desc_vector")); + new HashSet( + Arrays.asList("_content_vector", "_name_vector", "_desc_vector", "_description_vector")); + + /** model id */ + public static final String MODEL_ID = "modelId"; + + /** model type */ + public static final String MODEL_TYPE = "modelType"; + + /** logo */ + public static final String LOGO = "logo"; /** text/event-stream */ public static final String STEAM_CONTENT_TYPE = "text/event-stream;charset=UTF-8"; @@ -142,6 +161,9 @@ public class SpgAppConstant { /** json/application */ public static final String JSON_CONTENT_TYPE = "application/json;charset=UTF-8"; + /** stream empty line */ + public static final String STREAM_EMPTY_LINE = "data: \n\n"; + /** stream end line */ public static final String STREAM_END_LINE = "data: [DONE]\n\n"; @@ -150,4 +172,43 @@ public class SpgAppConstant { /** stream error line */ public static final String STREAM_ERROR_LINE = "data: [ERROR]\n\n"; + + /** stream think end tag */ + public static final String STREAM_THINK_END_TAG = ""; + + /** visibility */ + public static final String VISIBILITY = "visibility"; + + /** customize */ + public static final String CUSTOMIZE = "customize"; + + /** mcp servers */ + public static final String MCP_SERVERS = "mcp_servers"; + + /** senior */ + public static final String SENIOR = "senior"; + + /** provider */ + public static final String PROVIDER = "provider"; + + /** app_id */ + public static final String APP_ID = "app_id"; + + /** web display inherited properties */ + public static final List DISPLAY_INHERIT_PROPERTIES = + Lists.newArrayList("name", "description"); + + public static final String NAME = "name"; + public static final String NAME_ZH = "名称"; + + public static final String DESCRIPTION = "description"; + public static final String DESCRIPTION_ZH = "描述"; + + public static final String CHAT = "chat"; + public static final String ENAME = "ename"; + public static final String HOST_ADDR = "host_addr"; + public static final String MAYA_HTTP = "maya_http"; + + /** account name check pattern */ + public static final Pattern ACCOUNT_PATTERN = Pattern.compile("^[a-zA-Z0-9_]{6,20}$"); } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/CommonUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/CommonUtils.java index eeb608883..8d35a83ae 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/CommonUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/CommonUtils.java @@ -81,6 +81,15 @@ public static String getTaskStorageFileKey( return getInstanceStorageFileKey(projectId, instanceId) + taskId + "_" + type + ".kag"; } + public static String getTaskStoragePathKey( + Long projectId, Long instanceId, Long taskId, String type) { + return getInstanceStorageFileKey(projectId, instanceId) + taskId + "_" + type + File.separator; + } + + public static String getTaskStorageFileKey(String path, String name) { + return path + name + ".kag"; + } + public static JSONObject getKagBuilderConfig( Project project, BuilderJob builderJob, String hostAddr) { JSONObject extension = JSON.parseObject(builderJob.getExtension()); @@ -135,6 +144,7 @@ public static JSONObject getKagBuilderConfig( JSONObject writer = new JSONObject(); writer.put(BuilderConstant.TYPE, BuilderConstant.KG_WRITER); + writer.put(BuilderConstant.PROJECT_ID, project.getId()); if (BuilderConstant.DELETE.equalsIgnoreCase(builderJob.getAction())) { writer.put(BuilderConstant.DELETE, true); } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/DateTimeUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/DateTimeUtils.java index e6af957b6..913c4bbbf 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/DateTimeUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/DateTimeUtils.java @@ -14,8 +14,13 @@ package com.antgroup.openspg.common.util; import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.List; public class DateTimeUtils { @@ -57,4 +62,69 @@ public static Date getTime(int past, boolean zeroHMS) { } return calendar.getTime(); } + + public static Date getDateWithoutMs() { + Calendar calendar = Calendar.getInstance(); + calendar.set(Calendar.MILLISECOND, 0); + return calendar.getTime(); + } + + /** + * Get the difference in months between two dates + * + * @param startTime + * @param endTime + * @return + */ + public static long getMonthDifferenceFromDates(String startTime, String endTime) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDate startDate = LocalDate.parse(startTime, formatter); + LocalDate endDate = LocalDate.parse(endTime, formatter); + return ChronoUnit.MONTHS.between(startDate, endDate) + 1; + } + + /** + * ` Get all dates between two dates + * + * @param startTime yyyy-MM-dd + * @param endTime yyyy-MM-dd + * @return Date List + */ + public static List getDatesBetweenRange(String startTime, String endTime) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + LocalDate startDate = LocalDate.parse(startTime, formatter); + LocalDate endDate = LocalDate.parse(endTime, formatter); + + List dates = new ArrayList<>(); + while (!startDate.isAfter(endDate)) { + dates.add(startDate.format(formatter)); + startDate = startDate.plusDays(1); + } + return dates; + } + + /** + * Get all months between two dates + * + * @param startTime yyyy-MM-dd + * @param endTime yyyy-MM-dd + * @return Month List + */ + public static List getMonthsBetweenRange(String startTime, String endTime) { + DateTimeFormatter inputFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM"); + + // 解析输入日期为 LocalDate + LocalDate startDate = LocalDate.parse(startTime, inputFormatter); + LocalDate endDate = LocalDate.parse(endTime, inputFormatter); + + List dates = new ArrayList<>(); + // 输出两个日期间的所有月份 + while (!startDate.isAfter(endDate)) { + dates.add(startDate.format(formatter)); + startDate = startDate.plusMonths(1); + } + return dates; + } } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java index ee83328bd..b380f039c 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.common.util; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; +import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -29,4 +29,27 @@ public static String toJsonString(Object object) { } return result; } + + /** + * Flatten the json object + * + * @param jsonObject + * @return + */ + public static JSONObject flatten(JSONObject jsonObject) { + JSONObject flattenedObject = new JSONObject(); + flatten(jsonObject, flattenedObject, ""); + return flattenedObject; + } + + private static void flatten(JSONObject jsonObject, JSONObject flattenedObject, String prefix) { + for (String key : jsonObject.keySet()) { + Object value = jsonObject.get(key); + if (value instanceof JSONObject) { + flatten((JSONObject) value, flattenedObject, prefix); + } else { + flattenedObject.put(prefix.isEmpty() ? key : prefix + "." + key, value); + } + } + } } diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/SchemaJsonUtils.java similarity index 97% rename from server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java rename to common/util/src/main/java/com/antgroup/openspg/common/util/SchemaJsonUtils.java index 7f796fb20..2959da2b0 100644 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/SchemaJsonUtils.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.api.facade; +package com.antgroup.openspg.common.util; import com.antgroup.openspg.core.schema.model.constraint.BaseConstraintItem; import com.antgroup.openspg.core.schema.model.constraint.ConstraintTypeEnum; @@ -39,6 +39,7 @@ import com.antgroup.openspg.core.schema.model.type.ConceptType; import com.antgroup.openspg.core.schema.model.type.EntityType; import com.antgroup.openspg.core.schema.model.type.EventType; +import com.antgroup.openspg.core.schema.model.type.IndexType; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; import com.antgroup.openspg.core.schema.model.type.StandardType; import com.google.gson.Gson; @@ -68,6 +69,7 @@ public class SchemaJsonUtils { .registerSubtype(EntityType.class, SPGTypeEnum.ENTITY_TYPE.name()) .registerSubtype(ConceptType.class, SPGTypeEnum.CONCEPT_TYPE.name()) .registerSubtype(EventType.class, SPGTypeEnum.EVENT_TYPE.name()) + .registerSubtype(IndexType.class, SPGTypeEnum.INDEX_TYPE.name()) .registerSubtype(StandardType.class, SPGTypeEnum.STANDARD_TYPE.name()) .registerSubtype(TextBasicType.class, BasicTypeEnum.TEXT.name()) .registerSubtype(LongBasicType.class, BasicTypeEnum.LONG.name()) diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/SimpleThreadFactory.java b/common/util/src/main/java/com/antgroup/openspg/common/util/SimpleThreadFactory.java new file mode 100644 index 000000000..4f0e9607f --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/SimpleThreadFactory.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +public class SimpleThreadFactory implements ThreadFactory { + private static final AtomicInteger POOL_NUMBER = new AtomicInteger(1); + private static final String SEPARATOR = "-"; + private final ThreadGroup group; + private final AtomicInteger threadNumber = new AtomicInteger(1); + private final String namePrefix; + + public SimpleThreadFactory(String namePrefix) { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + this.namePrefix = namePrefix + SEPARATOR + POOL_NUMBER.getAndIncrement() + SEPARATOR; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); + if (t.isDaemon()) { + t.setDaemon(false); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/TraceCallableWrapper.java b/common/util/src/main/java/com/antgroup/openspg/common/util/TraceCallableWrapper.java new file mode 100644 index 000000000..0a20482ea --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/TraceCallableWrapper.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util; + +import java.util.Map; +import java.util.concurrent.Callable; +import org.slf4j.MDC; + +public abstract class TraceCallableWrapper implements Callable { + + private Map contextMap = null; + private long tid = Thread.currentThread().getId(); + + public TraceCallableWrapper() { + if (MDC.getMDCAdapter() != null) { + this.contextMap = MDC.getCopyOfContextMap(); + } + } + + @Override + public T call() throws Exception { + if (contextMap != null) { + MDC.setContextMap(contextMap); + } + try { + return doCall(); + } finally { + if (MDC.getMDCAdapter() != null && Thread.currentThread().getId() != tid) { + MDC.clear(); + } + } + } + + protected abstract T doCall() throws Exception; + + public static TraceCallableWrapper of(Callable callable) { + return new TraceCallableWrapper() { + @Override + protected T doCall() throws Exception { + return callable.call(); + } + }; + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/constants/CommonConstant.java b/common/util/src/main/java/com/antgroup/openspg/common/util/constants/CommonConstant.java index 144050f63..8841672b9 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/constants/CommonConstant.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/constants/CommonConstant.java @@ -19,4 +19,6 @@ public class CommonConstant { public static final String TUNNEL_ENDPOINT = "TunnelEndpoint"; public static final String KAG = "kag."; + + public static final String KGFABRIC_GRAPH_STORE = "kgfabric"; } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/AdvancedTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/AdvancedTypeEnum.java new file mode 100644 index 000000000..c78c7603f --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/AdvancedTypeEnum.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public enum AdvancedTypeEnum { + ENTITY_TYPE("EntityType", "实体类型"), + INDEX_TYPE("IndexType", "索引类型实体"), + CONCEPT_TYPE("ConceptType", "概念类型"), + EVENT_TYPE("EventType", "事件类型"), + STANDARD_TYPE("StandardType", "标准类型"); + + private String code; + private String desc; + + AdvancedTypeEnum(String code, String desc) { + this.code = code; + this.desc = desc; + } + + public static AdvancedTypeEnum toEnumByCode(String code) { + for (AdvancedTypeEnum advancedTypeEnum : values()) { + if (advancedTypeEnum.getCode().equalsIgnoreCase(code)) { + return advancedTypeEnum; + } + } + throw new IllegalArgumentException("code is illegal"); + } + + public static AdvancedTypeEnum toEnum(String typeName) { + for (AdvancedTypeEnum advancedTypeEnum : values()) { + if (advancedTypeEnum.name().equalsIgnoreCase(typeName)) { + return advancedTypeEnum; + } + } + throw new IllegalArgumentException("typeName is illegal"); + } + + private static final Map> CONVERSION_RULES = + Collections.unmodifiableMap( + new HashMap>() { + { + put( + AdvancedTypeEnum.INDEX_TYPE, + Collections.unmodifiableSet( + new java.util.HashSet() { + { + add(AdvancedTypeEnum.ENTITY_TYPE); + } + })); + } + }); + + public static boolean canConvert(AdvancedTypeEnum old, AdvancedTypeEnum newType) { + return old != null + && newType != null + && (old.equals(newType) + || newType.equals(AdvancedTypeEnum.INDEX_TYPE) + || CONVERSION_RULES.getOrDefault(old, Collections.emptySet()).contains(newType)); + } + + public String getCode() { + return code; + } + + public String getDesc() { + return desc; + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/BasicTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/BasicTypeEnum.java new file mode 100644 index 000000000..8c540566c --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/BasicTypeEnum.java @@ -0,0 +1,48 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +import lombok.Getter; + +@Getter +public enum BasicTypeEnum { + TEXT("Text", "文本"), + INTEGER("Integer", "整型"), + FLOAT("Float", "浮点数"); + + private String code; + private String desc; + + BasicTypeEnum(String code, String desc) { + this.code = code; + this.desc = desc; + } + + public static BasicTypeEnum getByCode(String code) { + for (BasicTypeEnum typeEnum : values()) { + if (typeEnum.getCode().equalsIgnoreCase(code)) { + return typeEnum; + } + } + throw new IllegalArgumentException("code is illegal"); + } + + public static boolean isBasicType(String code) { + for (BasicTypeEnum typeEnum : values()) { + if (typeEnum.getCode().equalsIgnoreCase(code)) { + return true; + } + } + return false; + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/FeedbackTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/FeedbackTypeEnum.java new file mode 100644 index 000000000..67902077c --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/FeedbackTypeEnum.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum FeedbackTypeEnum { + + /** 问答 */ + QA; + + public static FeedbackTypeEnum getReactionTypeEnum(String reactionType) { + for (FeedbackTypeEnum feedbackTypeEnum : FeedbackTypeEnum.values()) { + if (feedbackTypeEnum.name().equals(reactionType)) { + return feedbackTypeEnum; + } + } + throw new IllegalArgumentException("Unsupported feedbackType:" + reactionType); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ModelTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ModelTypeEnum.java new file mode 100644 index 000000000..e593fe509 --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ModelTypeEnum.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum ModelTypeEnum { + LLM("chat", "大语言模型"), + EMBEDDING("embedding", "向量模型"), + RERANKER("reranker", "重排模型"), + TEXT_TO_IMAGE("text-to-image", "图片生成"), + IMAGE_TO_IMAGE("image-to-image", "视觉模型"), + SPEECH_TO_TEXT("speech-to-text", "语音识别"), + TEXT_TO_VIDEO("text-to-video", "视频生成"), + ; + + private String code; + private String desc; + + ModelTypeEnum(String code, String desc) { + this.code = code; + this.desc = desc; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public static ModelTypeEnum getByCode(String code) { + for (ModelTypeEnum value : values()) { + if (value.code.equals(code)) { + return value; + } + } + throw new IllegalArgumentException("Unsupported model type:" + code); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/PageModeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/PageModeEnum.java new file mode 100644 index 000000000..6d9e067fe --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/PageModeEnum.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum PageModeEnum { + + /** ALL */ + ALL, + + /** SINGLE */ + SINGLE, + ; + + /** + * get by name + * + * @param name + * @return + */ + public static PageModeEnum getByName(String name) { + for (PageModeEnum mode : values()) { + if (mode.name().equals(name)) { + return mode; + } + } + throw new IllegalArgumentException("Unsupported page model:" + name); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ProjectTagEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ProjectTagEnum.java new file mode 100644 index 000000000..f9f1f84d1 --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ProjectTagEnum.java @@ -0,0 +1,18 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum ProjectTagEnum { + LOCAL, + PUBLIC_NET +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ReactionTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ReactionTypeEnum.java new file mode 100644 index 000000000..09bc75bbd --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ReactionTypeEnum.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum ReactionTypeEnum { + DEFAULT, + UP, + DOWN; + + public static ReactionTypeEnum getReactionTypeEnum(String reactionType) { + for (ReactionTypeEnum reactionTypeEnum : ReactionTypeEnum.values()) { + if (reactionTypeEnum.name().equals(reactionType)) { + return reactionTypeEnum; + } + } + throw new IllegalArgumentException("Unsupported reactionType:" + reactionType); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ResourceTagEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ResourceTagEnum.java index a04dece4f..544ef6f6b 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ResourceTagEnum.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/ResourceTagEnum.java @@ -14,11 +14,14 @@ public enum ResourceTagEnum { - /** project */ - PROJECT, - /** platform */ PLATFORM, + + /** app */ + APP, + + /** knowledge base */ + KNOWLEDGE_BASE, ; /** diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/SessionTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/SessionTypeEnum.java new file mode 100644 index 000000000..eee68dba8 --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/SessionTypeEnum.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum SessionTypeEnum { + /** normal */ + NORMAL, + + /** debug */ + DEBUG, + + /** api */ + API; + + public static SessionTypeEnum getByName(String name) { + for (SessionTypeEnum type : values()) { + if (type.name().equals(name)) { + return type; + } + } + throw new IllegalArgumentException("unknown session type: " + name); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/StatisticsTypeEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/StatisticsTypeEnum.java new file mode 100644 index 000000000..2fb8c784a --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/StatisticsTypeEnum.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum StatisticsTypeEnum { + + /** USER_STATISTICS */ + USER_STATISTICS, + + /** NORMAL_CHAT_STATISTICS */ + NORMAL_CHAT_STATISTICS, + + /** DEBUG_CHAT_STATISTICS */ + DEBUG_CHAT_STATISTICS, + + /** API_CHAT_STATISTICS */ + API_CHAT_STATISTICS, + + /** CHAT_STATISTICS */ + CHAT_STATISTICS, + + /** TOKENS_STATISTICS */ + TOKENS_STATISTICS, + + /** FEEDBACK_UP_STATISTICS */ + FEEDBACK_UP_STATISTICS, + + /** FEEDBACK_DOWN_STATISTICS */ + FEEDBACK_DOWN_STATISTICS; + + /** + * get enum by statistics type + * + * @param statisticsType + * @return + */ + public static StatisticsTypeEnum getStatisticsType(String statisticsType) { + for (StatisticsTypeEnum tagEnum : StatisticsTypeEnum.values()) { + if (tagEnum.name().equals(statisticsType)) { + return tagEnum; + } + } + + throw new IllegalArgumentException("Unsupported statistics type:" + statisticsType); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/enums/VisibilityEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/VisibilityEnum.java new file mode 100644 index 000000000..d78c0b07c --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/enums/VisibilityEnum.java @@ -0,0 +1,37 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.enums; + +public enum VisibilityEnum { + + /** public read */ + PUBLIC_READ, + + /** private */ + PRIVATE; + + /** + * get visibility enum by visibility + * + * @param visibility + * @return + */ + public static VisibilityEnum getVisibilityEnum(String visibility) { + for (VisibilityEnum visibilityEnum : VisibilityEnum.values()) { + if (visibilityEnum.name().equals(visibility)) { + return visibilityEnum; + } + } + throw new IllegalArgumentException("Unsupported visibility:" + visibility); + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/exception/ChatException.java b/common/util/src/main/java/com/antgroup/openspg/common/util/exception/ChatException.java new file mode 100644 index 000000000..9f17f7364 --- /dev/null +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/exception/ChatException.java @@ -0,0 +1,103 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.common.util.exception; + +import com.antgroup.openspg.common.util.exception.message.Message; +import java.util.Arrays; + +public class ChatException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + protected String code; + protected String message; + protected String[] params; + + public ChatException(Message errorMessage) { + super(errorMessage.getMessage()); + this.code = errorMessage.getCode(); + this.message = errorMessage.getMessage(); + } + + public ChatException(Throwable t, Message errorMessage) { + super(errorMessage.getMessage(), t); + this.code = errorMessage.getCode(); + this.message = errorMessage.getMessage(); + } + + public ChatException(Message errorMessage, String... params) { + super(String.format(errorMessage.getMessage(), params)); + this.code = errorMessage.getCode(); + this.message = String.format(errorMessage.getMessage(), params); + this.params = params; + } + + public ChatException(Throwable t, Message errorMessage, String... params) { + super(String.format(errorMessage.getMessage(), params), t); + this.code = errorMessage.getCode(); + this.message = String.format(errorMessage.getMessage(), params); + this.params = params; + } + + public ChatException(String code, String message, String... params) { + super(String.format(message, params)); + this.code = code; + this.message = String.format(message, params); + this.params = params; + } + + public ChatException(String code, String message) { + super(message); + this.code = code; + this.message = message; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String[] getParams() { + return params; + } + + public void setParams(String[] params) { + this.params = params; + } + + @Override + public String toString() { + return "ChatException{" + + "code='" + + code + + '\'' + + ", message='" + + message + + '\'' + + ", params=" + + Arrays.toString(params) + + '}'; + } +} diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/exception/message/SpgMessageEnum.java b/common/util/src/main/java/com/antgroup/openspg/common/util/exception/message/SpgMessageEnum.java index e58a76efe..1993da82d 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/exception/message/SpgMessageEnum.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/exception/message/SpgMessageEnum.java @@ -14,18 +14,54 @@ public enum SpgMessageEnum implements Message { KAG_ERROR("KAG_0001", "Some error, please check /home/admin/logs/antspg/common-error.log"), + KAG_MISS_AUTH("KAG_0002", "Access is denied due to lack of authentication information"), + KAG_NO_PERMISSION("KAG_0003", "Access is denied, no permission to access"), + LOGIN_USER_NOT_EXIST("LOGIN_0001", "User does not exist"), LOGIN_USER_NOT_LOGIN("LOGIN_0002", "User not logged in"), LOGIN_SUPER_PASSWORD_NOT_CHANGE( "LOGIN_0003", "The default password of the system needs to be changed"), - PROJECT_MEMBER_NOT_EXIST("PRO_0001", "User does not have project member permission"), - PROJECT_NOT_EXIST("PRO_0002", "Project does not exist"), - PROJECT_NAMESPACE_NOT_EXIST("PRO_0003", "Namespace does not exist"), + KB_MEMBER_NOT_EXIST("KB_0001", "User is not a member of the knowledge base"), + KB_NOT_EXIST("KB_0002", "Knowledge base not exist"), + KB_NAMESPACE_NOT_EXIST("KB_0003", "Namespace corresponds to a non-existent knowledge base"), + KB_USED_ALREADY_CANNOT_DELETE("KB_0004", "Cannot delete used Knowledge base. Reference App:【%s】"), + KB_NOT_OWNER("KB_0005", "User is not a owner of the knowledge base"), + KB_NAME_EXIST("KB_0006", "Knowledge base name already exists"), SCHEMA_CHANGE_NODE_TYPE("SCHEMA_0001", "Schema advanced node type cannot be changed"), - SCHEMA_CHANGE_PROPERTY_TYPE("SCHEMA_0001", "Schema property type cannot be changed"), - SCHEMA_CHANGE_HYPERNYM_PREDICATE("SCHEMA_0001", "Schema hypernymPredicate cannot be changed"), + SCHEMA_CHANGE_PROPERTY_TYPE("SCHEMA_0002", "Schema property type cannot be changed"), + SCHEMA_CHANGE_HYPERNYM_PREDICATE("SCHEMA_0003", "Schema hypernymPredicate cannot be changed"), + SCHEMA_CHANGE_BASIC_TYPE( + "SCHEMA_0004", + "Schema advanced type[%s]'s property[%s] are built-in attributes, and currently only Text type is allowed"), + + MODEL_PROVIDER_NAME_EXIST("MODEL_0001", "Provider name already exists"), + MODEL_NAME_EXIST("MODEL_0002", "Model name already exists"), + MODEL_PARAM_TYPE_NOT_SUPPORT("MODEL_0003", "Param type not support"), + MODEL_PUBLIC_ONLY_ONE_ALLOWED("MODEL_0004", "Public model only allowed one"), + MODEL_USED_ALREADY_CANNOT_DELETE("MODEL_0005", "Cannot delete used model%s"), + MODEL_NOT_PERMISSION("MODEL_0006", "No permission of the model"), + + APP_NAME_EXIST("APP_0001", "Application name already exists"), + APP_NOT_FOUND("APP_0002", "Application not found"), + APP_USED_ALREADY_CANNOT_DELETE("APP_0003", "Cannot delete used application"), + APP_NOT_PERMISSION("APP_0004", "No permission of the application"), + APP_NOT_OWNER("APP_0005", "User is not a owner of the application"), + APP_NOT_SET_TEMPLATE("APP_0006", "The application has no session template"), + APP_NOT_SET_KB("APP_0007", "The application has no knowledge base"), + APP_USED_ALREADY_CANNOT_UPDATE_ALIAS( + "APP_0008", "The alias cannot be modified while the app is in use"), + APP_ALIAS_EXIST("APP_0009", "Application alias already exists"), + + ACCESS_TOKEN_INVALID("ACCESS_TOKEN_0001", "Access token is invalid"), + ACCESS_TOKEN_FORBID("ACCESS_TOKEN_0002", "Access token is forbidden"), + CHAT_ACCESS_TOKEN_INVALID("CHAT_0001", "Chat access token is invalid"), + CHAT_ACCESS_TOKEN_FORBID("CHAT_0002", "Chat access token is forbidden"), + SESSION_NOT_PERMISSION("SESSION_0001", "No permission of the session"), + + REACTION_TYPE_ERROR("FEED_BACK_0001", "No changes were found in the reaction types"), + CANNOT_EDITED_ERROR("FEED_BACK_0002", "Feedback cannot be edited"), ; SpgMessageEnum(String code, String message) { @@ -41,15 +77,7 @@ public String getCode() { return code; } - public void setCode(String code) { - this.code = code; - } - public String getMessage() { return message; } - - public void setMessage(String message) { - this.message = message; - } } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jGraphUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jGraphUtils.java index 4bec9c0b5..faa986506 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jGraphUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jGraphUtils.java @@ -70,6 +70,10 @@ public void initializeSchema(List schemaTypes, int vectorDimensions neo4jIndex.createVectorIndex( schemaType.getName(), property.getName(), vectorDimensions); break; + case SPARSE_VECTOR: + case TEXT_AND_SPARSE_VECTOR: + log.info("Neo4j doesn't support sparse vector index: {}", indexType); + break; default: log.info("Undefined IndexTypeEnum {}", indexType); } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jIndexUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jIndexUtils.java index 9bbf35f38..02cf5de74 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jIndexUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/neo4j/Neo4jIndexUtils.java @@ -364,6 +364,18 @@ public List vectorSearch( } } + public List customSearch(@NonNull String customQuery) { + try (Session session = driver.session(SessionConfig.forDatabase(database))) { + return session.readTransaction( + tx -> { + Result result = tx.run(customQuery); + List records = result.list(); + tx.commit(); + return records; + }); + } + } + public Map> getVectorIndexMeta() { String query = "SHOW VECTOR INDEX"; try (Session session = driver.session(SessionConfig.forDatabase(database))) { @@ -418,6 +430,7 @@ private static boolean shouldCreateTextIndex(@NonNull Property property) { switch (indexType) { case TEXT: case TEXT_AND_VECTOR: + case TEXT_AND_SPARSE_VECTOR: return true; default: return false; diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PemjaUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PemjaUtils.java index 2f7f0c653..01ff07fcc 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PemjaUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PemjaUtils.java @@ -15,9 +15,19 @@ import com.alibaba.fastjson.JSONObject; import com.antgroup.openspg.common.util.Md5Utils; +import com.antgroup.openspg.common.util.RetryerUtil; +import com.antgroup.openspg.common.util.SimpleThreadFactory; +import com.antgroup.openspg.common.util.TraceCallableWrapper; import com.antgroup.openspg.common.util.pemja.model.PemjaConfig; +import com.github.rholder.retry.Retryer; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; @@ -28,59 +38,168 @@ @Slf4j public class PemjaUtils { + private static Retryer retry = RetryerUtil.getRetryer(50L, 2L, 3); + public static Object invoke(PemjaConfig config, Object... input) { - String uniqueKey = config.getClassName() + "_" + Md5Utils.md5Of(UUID.randomUUID().toString()); + String md5 = Md5Utils.md5Of(UUID.randomUUID().toString()); + String uniqueKey = config.getClassName() + "_" + md5; log.info( "PemjaUtils.invoke uniqueKey:{} config:{} input:{}", uniqueKey, JSONObject.toJSONString(config), JSONObject.toJSONString(input)); + AtomicReference result = new AtomicReference<>(); + AtomicReference throwable = new AtomicReference<>(); + try { + retry.call( + () -> { + log.info("PemjaUtils.invokeMethod uniqueKey:{}", uniqueKey); + ThreadPoolExecutor executor = + new ThreadPoolExecutor( + 1, + 1, + 5L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), + new SimpleThreadFactory("pemja")); + executor.allowCoreThreadTimeOut(true); + Long start = System.currentTimeMillis(); + Future future = + executor.submit( + TraceCallableWrapper.of( + () -> { + PythonInterpreter interpreter = null; + try { + interpreter = getPythonInterpreter(config, uniqueKey); + return interpreter.invokeMethod(uniqueKey, config.getMethod(), input); + } finally { + if (interpreter != null) { + interpreter.close(); + log.info( + "PemjaUtils.invokeMethod interpreter close uniqueKey:" + + uniqueKey); + } + } + })); + try { + result.set(future.get(9, TimeUnit.MINUTES)); + log.info( + "PemjaUtils.invoke succeed cons:{} uniqueKey:{} config:{}", + System.currentTimeMillis() - start, + uniqueKey, + JSONObject.toJSONString(config)); + return true; + } catch (TimeoutException e) { + log.error("PemjaUtils.invoke TimeoutException uniqueKey:" + uniqueKey, e); + future.cancel(true); + RuntimeException exception = + new RuntimeException("PemjaUtils.invoke TimeoutException:" + e.getMessage(), e); + throwable.set(exception); + throw exception; + } catch (Exception e) { + log.error( + String.format( + "PemjaUtils.invoke Exception cons:%s uniqueKey:%s config:%s", + System.currentTimeMillis() - start, + uniqueKey, + JSONObject.toJSONString(config)), + e); + RuntimeException exception = + new RuntimeException("PemjaUtils.invoke Exception:" + e.getMessage(), e); + throwable.set(exception); + return true; + } finally { + executor.shutdown(); + log.info( + String.format( + "PemjaUtils.invoke executor shutdown: activeCount=%s", + executor.getActiveCount())); + } + }); + } catch (Exception e) { + log.error("PemjaUtils.invoke retry Exception uniqueKey:" + uniqueKey, e); + throw new RuntimeException("PemjaUtils.invoke retry Exception:" + e.getMessage(), e); + } + if (result.get() == null) { + throw throwable.get(); + } + return result.get(); + } + + public static void invokeAsync(PemjaConfig config, Object... input) { + String uniqueKey = config.getClassName() + "_" + Md5Utils.md5Of(UUID.randomUUID().toString()); + log.info( + "PemjaUtils.invokeAsync uniqueKey:{} config:{} input:{}", + uniqueKey, + JSONObject.toJSONString(config), + JSONObject.toJSONString(input)); PythonInterpreter interpreter = null; try { + Long start = System.currentTimeMillis(); interpreter = getPythonInterpreter(config, uniqueKey); - Object result = interpreter.invokeMethod(uniqueKey, config.getMethod(), input); + interpreter.invokeMethod(uniqueKey, config.getMethod(), input); log.info( - "PemjaUtils.invoke succeed uniqueKey:{} result:{}", + "PemjaUtils.invokeAsync succeed cons:{} uniqueKey:{} config:{}", + System.currentTimeMillis() - start, uniqueKey, - JSONObject.toJSONString(result)); - return result; + JSONObject.toJSONString(config)); } finally { if (interpreter != null) { interpreter.close(); + log.info("PemjaUtils.invokeAsync interpreter close uniqueKey:" + uniqueKey); } } } private static PythonInterpreter getPythonInterpreter(PemjaConfig config, String uniqueKey) { + log.info("PemjaUtils.getPythonInterpreter start uniqueKey:{}", uniqueKey); PythonInterpreter interpreter = newPythonInterpreter(config.getPythonExec(), config.getPythonPaths()); + log.info("PemjaUtils.getPythonInterpreter newPythonInterpreter uniqueKey:{}", uniqueKey); if (config.getProjectId() != null) { interpreter.exec("from kag.bridge.spg_server_bridge import init_kag_config"); interpreter.exec( String.format( "init_kag_config(\"%s\",\"%s\")", config.getProjectId(), config.getHostAddr())); } + Long start = System.currentTimeMillis(); + log.info("PemjaUtils.getPythonInterpreter before exec uniqueKey:{}", uniqueKey); interpreter.exec( String.format("from %s import %s", config.getModulePath(), config.getClassName())); + log.info( + String.format( + "PemjaUtils.getPythonInterpreter exec1 uniqueKey：%s cost:%s", + uniqueKey, System.currentTimeMillis() - start)); interpreter.exec( String.format( "%s=%s(%s)", uniqueKey, config.getClassName(), paramToPythonString(config.getParams(), config.getParamsPrefix()))); + log.info( + String.format( + "PemjaUtils.getPythonInterpreter exec2 uniqueKey：%s cost:%s", + uniqueKey, System.currentTimeMillis() - start)); return interpreter; } public static PythonInterpreter newPythonInterpreter(String pythonExec, String pythonPaths) { + log.info( + String.format( + "PemjaUtils.newPythonInterpreter start pythonExec：%s pythonPaths:%s", + pythonExec, pythonPaths)); PythonInterpreterConfig.PythonInterpreterConfigBuilder builder = PythonInterpreterConfig.newBuilder(); + log.info("PemjaUtils.newPythonInterpreter newBuilder"); if (StringUtils.isNotBlank(pythonExec)) { builder.setPythonExec(pythonExec); } + log.info("PemjaUtils.newPythonInterpreter setPythonExec"); if (StringUtils.isNotBlank(pythonPaths)) { String[] pythonPathList = pythonPaths.split(";"); builder.addPythonPaths(pythonPathList); } + log.info("PemjaUtils.newPythonInterpreter addPythonPaths"); return new PythonInterpreter(builder.build()); } diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PythonInvokeMethod.java b/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PythonInvokeMethod.java index 5f9f43e6b..e9672b6cf 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PythonInvokeMethod.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/pemja/PythonInvokeMethod.java @@ -17,6 +17,12 @@ public enum PythonInvokeMethod { BRIDGE_READER("bridge.spg_server_bridge", "SPGServerBridge", "run_reader", ""), BRIDGE_SCANNER("bridge.spg_server_bridge", "SPGServerBridge", "run_scanner", ""), BRIDGE_COMPONENT("bridge.spg_server_bridge", "SPGServerBridge", "run_component", ""), + BRIDGE_GET_LLM_TOKEN_INFO( + "bridge.spg_server_bridge", "SPGServerBridge", "get_llm_token_info", ""), + BRIDGE_GET_INDEX_MANAGER_NAMES( + "bridge.spg_server_bridge", "SPGServerBridge", "get_index_manager_names", ""), + BRIDGE_GET_INDEX_MANAGER_INFO( + "bridge.spg_server_bridge", "SPGServerBridge", "get_index_manager_info", ""), BRIDGE_LLM_CHECKER("bridge.spg_server_bridge", "SPGServerBridge", "run_llm_config_check", ""), BRIDGE_VECTORIZER_CHECKER( "bridge.spg_server_bridge", "SPGServerBridge", "run_vectorizer_config_check", ""), diff --git a/dev/release/mysql/sql/initdb.sql b/dev/release/mysql/sql/initdb.sql index e9620952d..ff3da1c91 100644 --- a/dev/release/mysql/sql/initdb.sql +++ b/dev/release/mysql/sql/initdb.sql @@ -13,23 +13,25 @@ use openspg; CREATE TABLE `kg_reason_session` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', - `project_id` bigint(20) unsigned NOT NULL COMMENT '项目ID', - `user_id` bigint(20) unsigned NOT NULL COMMENT '用户ID', + `project_id` bigint(20) unsigned NOT NULL COMMENT '项目ID,新版本会话存放appId', + `user_id` bigint(20) unsigned NOT NULL COMMENT '字段已废弃', `name` varchar(1024) NOT NULL COMMENT '会话名称', `description` longtext DEFAULT NULL COMMENT '会话描述信息', - `gmt_create` timestamp NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', - `gmt_modified` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp() COMMENT '修改时间', - PRIMARY KEY (`id`), - KEY `idx_project_id` (`project_id`,`id`) -) DEFAULT CHARSET=utf8mb4 COMMENT='图谱推理任务会话表'; + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `user_no` varchar(255) DEFAULT NULL COMMENT '用户工号', + `type` varchar(64) NOT NULL DEFAULT 'NORMAL' COMMENT '会话类型：DEBUG,NORMAL', + PRIMARY KEY(`id`), + KEY `idx_project_id`(`project_id`, `id`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '图谱推理任务会话表'; CREATE TABLE `kg_reason_task` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', - `project_id` bigint(20) unsigned NOT NULL COMMENT '项目ID', - `user_id` bigint(20) unsigned NOT NULL COMMENT '用户ID', + `project_id` bigint(20) unsigned NOT NULL COMMENT '项目ID，新版本会话存放appId', + `user_id` bigint(20) unsigned NOT NULL COMMENT '字段已废弃', `session_id` bigint(20) unsigned NOT NULL COMMENT '会话ID', - `gmt_create` timestamp NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', - `gmt_modified` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp() COMMENT '修改时间', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', `mark` varchar(16) DEFAULT 'NULL' COMMENT '收藏状态', `status` varchar(32) DEFAULT NULL COMMENT '状态', `dsl` longtext DEFAULT NULL COMMENT 'DSL执行语句', @@ -40,11 +42,13 @@ CREATE TABLE `kg_reason_task` ( `result_nodes` longtext DEFAULT NULL COMMENT '执行结果，点数据', `result_edges` longtext DEFAULT NULL COMMENT '执行结果，边数据', `result_paths` longtext DEFAULT NULL COMMENT '执行结果，路径数据', - PRIMARY KEY (`id`), - KEY `idx_session_id_id` (`session_id`,`id`), - KEY `idx_project_user_mark` (`project_id`,`user_id`,`mark`), - KEY `idx_user_mark_id` (`user_id`,`mark`,`id`) -) DEFAULT CHARSET=utf8mb4 COMMENT='图谱推理任务表'; + `user_no` varchar(255) DEFAULT NULL COMMENT '用户工号', + `reaction_type` varchar(64) DEFAULT NULL COMMENT '反应类型：UP/DOWN', + PRIMARY KEY(`id`), + KEY `idx_session_id_id`(`session_id`, `id`), + KEY `idx_project_userno_mark`(`project_id`, `user_no`, `mark`), + KEY `idx_userno_mark_id`(`user_no`, `mark`, `id`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '图谱推理任务表'; CREATE TABLE `kg_reason_tutorial` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -80,6 +84,7 @@ CREATE TABLE `kg_builder_job` ( `life_cycle` varchar(64) DEFAULT NULL COMMENT '执行周期类型', `action` varchar(64) DEFAULT NULL COMMENT '数据操作类型', `computing_conf` longtext DEFAULT NULL COMMENT '计算引擎配置', + `retrievals` varchar(512) DEFAULT NULL COMMENT '索引列表', PRIMARY KEY (`id`), KEY `idx_project_id` (`project_id`), KEY `idx_task_id` (`task_id`) @@ -183,8 +188,8 @@ CREATE TABLE `kg_data_source`( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', `gmt_create` timestamp NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', `gmt_modified` timestamp NOT NULL DEFAULT current_timestamp() on update current_timestamp () COMMENT '修改时间', - `create_user` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', - `update_user` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + `create_user` varchar(32) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `update_user` varchar(32) NOT NULL DEFAULT 'system' COMMENT '修改用户', `status` varchar(64) NOT NULL DEFAULT 'ENABLE' COMMENT '状态', `remark` varchar(1024) DEFAULT NULL COMMENT '描述', `type` varchar(64) NOT NULL DEFAULT 'MYSQL' COMMENT '数据源类型', @@ -200,3 +205,30 @@ UNIQUE KEY `uk_db_name` (`db_name`) ) DEFAULT CHARSET=utf8mb4 COMMENT='数据源管理表'; +CREATE TABLE `kg_retrieval` +( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` timestamp NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT current_timestamp() on update current_timestamp () COMMENT '修改时间', + `create_user` varchar(32) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `update_user` varchar(32) NOT NULL DEFAULT 'system' COMMENT '修改用户', + `type` varchar(64) NOT NULL DEFAULT 'KAG' COMMENT '类型', + `status` varchar(64) NOT NULL DEFAULT 'ENABLE' COMMENT '状态', + `is_default` varchar(8) NOT NULL DEFAULT 'N' comment '是否默认选中,取值：Y，N', + `name` varchar(128) NOT NULL COMMENT '名称', + `chinese_name` varchar(128) DEFAULT NULL COMMENT '中文名称', + `schema_desc` longtext DEFAULT NULL COMMENT 'schema描述', + `scenarios_desc` longtext DEFAULT NULL COMMENT '应用场景描述', + `cost_desc` longtext DEFAULT NULL COMMENT '成本描述', + `method_desc` longtext DEFAULT NULL COMMENT '索引方法描述', + `extractor_desc` longtext DEFAULT NULL COMMENT '索引抽取描述', + `retriever_desc` longtext DEFAULT NULL COMMENT '索引召回描述', + `module_path` varchar(128) NOT NULL DEFAULT '' COMMENT '索引类路径', + `class_name` varchar(128) NOT NULL DEFAULT '' COMMENT '索引类名称', + `method` varchar(128) NOT NULL DEFAULT '' COMMENT '索引类方法', + `extension` longtext DEFAULT NULL COMMENT '扩展信息', + `config` longtext DEFAULT NULL COMMENT '配置信息', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_path_class_method` (`module_path`,`class_name`,`method`), + UNIQUE KEY `uk_name` (`name`) +) DEFAULT CHARSET=utf8mb4 COMMENT='索引类配置表'; diff --git a/dev/release/mysql/sql/openspg-initdb.sql b/dev/release/mysql/sql/openspg-initdb.sql index 9cdac1fcb..7cebcf02c 100644 --- a/dev/release/mysql/sql/openspg-initdb.sql +++ b/dev/release/mysql/sql/openspg-initdb.sql @@ -12,21 +12,22 @@ use openspg; CREATE TABLE `kg_project_info` ( -`id` bigint(20) NOT NULL AUTO_INCREMENT comment '主键', -`name` varchar(255) NOT NULL comment '项目名称', -`description` varchar(1024) DEFAULT NULL comment '项目描述信息', -`status` varchar(20) NOT NULL DEFAULT 'INVALID' comment 'DELETE:删除 VALID:有效 INVALID：无效', -`gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP comment '创建时间', -`gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP comment '修改时间', -`namespace` varchar(64) NOT NULL DEFAULT '' comment '命名空间', -`biz_domain_id` bigint(20) DEFAULT NULL comment '业务域主键', -`config` text DEFAULT NULL comment '项目配置信息', -PRIMARY KEY(`id`), -UNIQUE KEY `uk_name`(`name`), -KEY `idx_biz_domain_id`(`biz_domain_id`) + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `name` varchar(255) NOT NULL COMMENT '项目名称', + `description` varchar(1024) DEFAULT NULL COMMENT '项目描述信息', + `status` varchar(20) NOT NULL DEFAULT 'INVALID' COMMENT 'DELETE:删除 VALID:有效 INVALID：无效', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `namespace` varchar(64) NOT NULL DEFAULT '' COMMENT '命名空间', + `biz_domain_id` bigint(20) DEFAULT NULL COMMENT '业务域主键', + `config` text DEFAULT NULL COMMENT '项目配置信息', + `visibility` varchar(64) DEFAULT 'PRIVATE' COMMENT '可见性：PRIVATE、PUBLIC_READ', + `tag` varchar(64) NOT NULL DEFAULT 'LOCAL' COMMENT '知识库标签：本地知识库（LOCAL）、公网知识库（PUBLIC-NET）', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_name`(`name`), + KEY `idx_biz_domain_id`(`biz_domain_id`) ) DEFAULT CHARSET = utf8mb4 COMMENT = '图谱项目信息表'; - CREATE TABLE `kg_biz_domain` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT comment '主键', `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP comment '创建时间', @@ -207,6 +208,7 @@ CREATE TABLE `kg_ontology_release` ( `version` int(11) NOT NULL comment '发布版本', `schema_view` longtext DEFAULT NULL comment '当前版本schema视图', `user_id` varchar(20) NOT NULL comment '发布人', +`user_no` varchar(255) NOT NULL COMMENT '发布人工号', `description` text NOT NULL comment '发布描述', `status` varchar(20) NOT NULL comment '状态', `change_procedure_id` text DEFAULT NULL comment '变更流程id', @@ -226,8 +228,8 @@ CREATE TABLE `kg_ontology_ext` ( `ext_type` varchar(64) NOT NULL comment '扩展类型：标签、回流、颜色', `field` varchar(64) NOT NULL comment '扩展属性所属域，比如区分用户', `config` mediumtext DEFAULT NULL comment '配置内容', - `creator` varchar(64) NOT NULL comment '创建者', - `modifier` varchar(64) NOT NULL comment '更新者', + `creator` varchar(255) NOT NULL comment '创建者', + `modifier` varchar(255) NOT NULL comment '更新者', `status` int(10) unsigned NOT NULL comment '状态 1：有效 0：无效', PRIMARY KEY(`id`), UNIQUE KEY `uk_id_type_field`(`resource_id`, `resource_type`, `ext_type`, `field`) @@ -301,6 +303,133 @@ CREATE TABLE `kg_role` ( UNIQUE KEY `uk_role_name`(`role_name`) ) DEFAULT CHARSET = utf8mb4 COMMENT = '平台角色表'; + +CREATE TABLE `kg_ref` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `name` varchar(512) NOT NULL COMMENT '关联名称', + `ref_id` varchar(255) NOT NULL COMMENT '关联id', + `ref_type` varchar(64) NOT NULL COMMENT '关联类型', + `refed_id` varchar(255) NOT NULL COMMENT '被关联Id', + `refed_type` varchar(64) NOT NULL COMMENT '被关联类型', + `config` longtext DEFAULT NULL COMMENT '配置详情', + `status` int(11) NOT NULL DEFAULT '1' COMMENT '状态：1 有效', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_id_type`(`ref_id`, `ref_type`, `refed_id`, `refed_type`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '关联表'; + +CREATE TABLE `kg_model_provider` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `name` varchar(32) NOT NULL COMMENT '名称', + `provider` varchar(32) NOT NULL COMMENT '供应商英文名称', + `status` varchar(32) NOT NULL COMMENT '状态 0：禁用，1：启用', + `page_mode` varchar(32) NOT NULL COMMENT '添加模型模式，ALL:全部模型，SINGLE:单个模型', + `model_type` varchar(255) NOT NULL COMMENT '支持模型类型(以逗号拼接)', + `logo` varchar(255) NOT NULL COMMENT '图标', + `tags` varchar(255) NOT NULL COMMENT '标签', + `params` longtext NOT NULL COMMENT '供应商参数json', + `creator` varchar(64) NOT NULL COMMENT '创建用户', + `modifier` varchar(64) NOT NULL COMMENT '修改用户', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_provider`(`provider`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '模型供应商表'; + +CREATE TABLE `kg_provider_param` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `provider` varchar(32) NOT NULL COMMENT '供应商英文名称', + `model_type` varchar(32) NOT NULL COMMENT '模型类型', + `model_name` varchar(32) DEFAULT NULL COMMENT '基础模型名称', + `params` longtext NOT NULL COMMENT '定制参数json', + `creator` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `modifier` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_provider_model_type`(`provider`, `model_type`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '供应商参数表'; + +CREATE TABLE `kg_model_detail` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `provider` varchar(32) NOT NULL COMMENT '供应商英文名称', + `type` varchar(32) NOT NULL COMMENT '模型类型', + `name` varchar(64) NOT NULL COMMENT '名称', + `description` varchar(255) DEFAULT NULL COMMENT '备注', + `params` longtext DEFAULT NULL COMMENT '模型参数json', + `creator` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `modifier` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_provider_type_name`(`provider`, `type`, `name`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '模型配置详情表'; + + +CREATE TABLE `kg_user_model` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `instance_id` varchar(64) NOT NULL COMMENT '实例id', + `visibility` varchar(32) NOT NULL COMMENT '可见性，PUBLIC_READ：公有、PRIVATE：私有', + `provider` varchar(32) NOT NULL COMMENT '供应商英文名称', + `name` varchar(64) NOT NULL COMMENT '唯一名称', + `config` longtext DEFAULT NULL COMMENT '模型配置字段json', + `creator` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `modifier` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_instanceid`(`instance_id`), + UNIQUE KEY `uk_provider_name_visibility`(`provider`, `name`, `visibility`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '用户模型表'; + +CREATE TABLE `kg_app` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `creator` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `modifier` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + `name` varchar(64) NOT NULL COMMENT '应用名称', + `description` text DEFAULT NULL COMMENT '备注', + `logo` varchar(1024) NOT NULL COMMENT '图标', + `config` longtext DEFAULT NULL COMMENT '应用参数', + `alias` varchar(32) DEFAULT NULL COMMENT '别名', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_name`(`name`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '应用表'; + +CREATE TABLE `kg_feedback` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `creator` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `modifier` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + `module_type` varchar(65) NOT NULL COMMENT '功能模块类型，QA:问答', + `one_category` varchar(255) NOT NULL COMMENT '一级类目', + `two_category` varchar(255) DEFAULT NULL COMMENT '二级类目', + `three_category` varchar(255) DEFAULT NULL COMMENT '三级类目', + `four_category` varchar(255) DEFAULT NULL COMMENT '四级类目', + `five_category` varchar(255) DEFAULT NULL COMMENT '五级类目', + `reaction_type` varchar(64) NOT NULL COMMENT '反应类型：DEFAULT/UP/DOWN', + `reason` longtext DEFAULT NULL COMMENT '点踩原因', + PRIMARY KEY(`id`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '反馈记录表'; + +CREATE TABLE `kg_statistics` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `creator` varchar(64) NOT NULL DEFAULT 'system' COMMENT '创建用户', + `modifier` varchar(64) NOT NULL DEFAULT 'system' COMMENT '修改用户', + `resource_tag` varchar(50) NOT NULL COMMENT '资源分类', + `resource_id` varchar(50) NOT NULL COMMENT '资源id：应用id、平台0', + `statistics_type` varchar(50) NOT NULL COMMENT '指标维度', + `statistics_date` varchar(50) NOT NULL COMMENT '统计具体时间', + `num` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '指标数量', + PRIMARY KEY(`id`), + UNIQUE KEY `uk_unique`(`resource_tag`, `resource_id`, `statistics_type`, `statistics_date`) +) DEFAULT CHARSET = utf8mb4 COMMENT = '统计汇总表'; + INSERT INTO kg_biz_domain (`id`,`gmt_create`,`gmt_modified`,`name`,`status`,`description`,`global_config`) VALUES(1,'2023-09-01 00:00:00','2023-09-01 00:00:00','defaultTenant','VALID','',null); INSERT INTO kg_ontology_entity (`id`,`original_id`,`name`,`name_zh`,`entity_category`,`layer`,`description`,`description_zh`,`status`,`with_index`,`scope`,`version`,`version_status`,`gmt_create`,`gmt_modified`,`transformer_id`,`operator_config`,`config`,`unique_name`) VALUES(1,1,'Thing','事物','ADVANCED','EXTENSION','Base class for all schema types, all of which inherit the type either directly or indirectly','所有schema类型的基类，它们都直接或者间接继承该类型','1','TRUE','PUBLIC',44,'ONLINE','2023-09-01 00:00:00','2023-09-01 00:00:00',0,null,null,'Thing'); @@ -335,6 +464,30 @@ INSERT INTO kg_role (`id`,`gmt_create`,`gmt_modified`,`role_name`,`permission_de INSERT INTO kg_resource_permission (`gmt_create`,`gmt_modified`,`user_no`,`resource_id`,`role_id`,`resource_tag`,`status`,`expire_date`) VALUES(now(),now(),'openspg',0,1,'PLATFORM','1',null); -INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(1,now(),now(),'0','KAG Support Model','KAG_SUPPORT_MODEL','1','[{"id":1,"vendor":"vllm","logo":"/img/logo/vllm.png","params":[{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"desc","cname":"desc","required":true,"formProps":{"allowClear":true,"placeholder":"Please enter remarks for partitioning."}}]},{"id":2,"vendor":"maas","logo":"/img/logo/maas.png","params":[{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"api_key","cname":"api_key ","required":true,"defaultValue":""},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"temperature","cname":"temperature","required":true,"formType":"number","defaultValue":0.7},{"ename":"stream","cname":"stream","required":true,"defaultValue":"False"},{"ename":"desc","cname":"desc","required":true,"formProps":{"allowClear":true,"placeholder":"Please enter remarks for partitioning."}}]},{"id":3,"vendor":"Ollama","logo":"/img/logo/ollama.png","params":[{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"desc","cname":"desc","required":true,"formProps":{"allowClear":true,"placeholder":"Please enter remarks for partitioning."}}]}]',1,null,null,'SYSTEM_CONFIG',''); -INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(2,now(),now(),'0','Global Configuration','KAG_CONFIG','1','',1,null,null,'CONFIG','admin'); -INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(3,now(),now(),'0','KAG Environment Configuration','KAG_ENV','1','{"configTitle":{"graph_store":{"id":1,"title":[{"ename":"database","cname":"database","required":true,"defaultValue":"kag","formProps":{"disabled":true}},{"ename":"password","cname":"password","required":true,"defaultValue":""},{"ename":"uri","cname":"uri","required":true,"defaultValue":""},{"ename":"user","cname":"user","required":true,"defaultValue":""}]},"vectorizer":{"id":2,"title":[{"ename":"type","cname":"type","required":true,"defaultValue":"openai","formProps":{"disabled":true}},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"api_key","cname":"api_key","required":true,"defaultValue":""}]},"prompt":{"id":3,"title":[{"ename":"biz_scene","cname":"biz_scene","required":true,"defaultValue":""},{"ename":"language","cname":"language","required":true,"defaultValue":""}]}}}',1,null,null,'SYSTEM_CONFIG',''); +INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(1,now(),now(),'0','KAG Support Model','KAG_SUPPORT_MODEL','1','[{"id":1,"vendor":"vllm","logo":"/img/logo/vllm.png","params":[{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"desc","cname":"desc","required":true,"formProps":{"allowClear":true,"placeholder":"Please enter remarks for partitioning."}}]},{"id":2,"vendor":"maas","logo":"/img/logo/maas.png","params":[{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"api_key","cname":"api_key ","required":true,"defaultValue":""},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"temperature","cname":"temperature","required":true,"formType":"number","defaultValue":0.7},{"ename":"stream","cname":"stream","required":true,"defaultValue":"False"},{"ename":"desc","cname":"desc","required":true,"formProps":{"allowClear":true,"placeholder":"Please enter remarks for partitioning."}}]},{"id":3,"vendor":"ollama","logo":"/img/logo/ollama.png","params":[{"ename":"base_url","cname":"base_url","required":true,"defaultValue":""},{"ename":"model","cname":"model","required":true,"defaultValue":""},{"ename":"desc","cname":"desc","required":true,"formProps":{"allowClear":true,"placeholder":"Please enter remarks for partitioning."}}]}]',1,null,null,'SYSTEM_CONFIG',''); +INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(3,now(),now(),'0','KAG Environment Configuration','KAG_ENV','1','{"showProfilePicture":false,"showUserConfig":true,"showLinks":false,"configTitle":{"graph_store":{"id":1,"title":[{"ename":"database","cname":"database","required":true,"inputType":"text","defaultValue":"kag","formProps":{"disabled":true}},{"ename":"password","cname":"password","required":true,"inputType":"text","defaultValue":"neo4j@openspg"},{"ename":"uri","cname":"uri","required":true,"inputType":"text","defaultValue":"neo4j://release-openspg-neo4j:7687"},{"ename":"user","cname":"user","required":true,"inputType":"text","defaultValue":"neo4j"}]},"prompt":{"id":3,"title":[{"ename":"language","cname":"语言","required":true,"defaultValue":"zh","inputType":"select","formProps":{"options":[{"label":"中文","value":"zh"},{"label":"英文","value":"en"}]}}]}}}',1,null,null,'SYSTEM_CONFIG',''); +INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(6,now(),now(),'0','APP_CHAT','APP_CHAT','1','[{"id":2,"cname":"推理问答","ename":"think_pipeline","logo":"/img/logo/modal_2.png","cdesc":"基于蚂蚁集团开源的专业领域知识服务框架KAG搭建的问答模板，擅长逻辑推理、数值计算等任务，可以协助解答相关问题、提供信息支持或进行数据分析","edesc":"The Q&A module, built on Ant Group\'s open-source KAG framework (a domain-specific knowledge service platform), demonstrates strong capabilities in logical reasoning and numerical computations. It effectively assists with problem-solving, delivers informational support, and performs data analysis tasks","thinking_enabled":{"defaultValue":true,"disabled":false}},{"id":4,"cname":"KAG Model问答","ename":"kag_thinker_pipeline","logo":"/img/logo/modal_2.png","thinking_enabled":{"defaultValue":true,"disabled":true},"cdesc":"基于蚂蚁集团开源的思考大模型及KAG搭建的问答模板，擅长逻辑推理、数值计算等任务，可以协助解答相关问题、提供信息支持或进行数据分析","edesc":"The Q&A template built on Ant Group\'s open-source thinking large model and KAG is excellent at tasks such as logical reasoning and numerical calculations. It can assist in answering relevant questions, providing information support, or conducting data analysis."}]', 1, NULL, NULL, 'SYSTEM_CONFIG', 'system'); +INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(8,now(),now(),'0','PROVIDER BASE INFO','PROVIDER_BASE_INFO','1','{"modelType":{"OpenAI_chat":"maas","Google_embedding":"openai","OpenAI_embedding":"openai","SILICONFLOW_chat":"maas","Ollama_chat":"ollama","DeepSeek_chat":"maas","BaiLian_embedding":"openai","ant_gpt_chat":"ant_gpt","maya_chat":"maya","Google_chat":"maas","OpenRouter_chat":"maas","ant_bailian_chat":"ant_bailian","ant_deepseek_chat":"ant_deepseek","maya_embedding":"maya","SILICONFLOW_embedding":"openai","zdfmng_stream_chat":"zdfmng_stream","BaiLian_chat":"maas","Ollama_embedding":"openai"}}',1,null,null,'SYSTEM_CONFIG','system'); +INSERT INTO kg_config (`id`,`gmt_create`,`gmt_modified`,`project_id`,`config_name`,`config_id`,`version`,`config`,`status`,`description`,`resource_id`,`resource_type`,`user_no`) VALUES(11,NOW(),NOW(),'0','FEEDBACK_REASON','FEEDBACK_REASON','1','[{"label":"理解","value":"UNDERSTAND","child":[{"value":"DISOBEYING_INSTRUCTIONS","label":"不理解问题/不服从指令"},{"value":"NOT_UNDERSTANDING_THE_CONTEXT","label":"不理解上下文"}]},{"label":"答案","value":"ANSWER","child":[{"value":"HALLUCINATIONS","label":"事实错误/幻觉"},{"value":"REASONING","label":"推理/计算错误"},{"value":"INCOMPLETE","label":"答案截断/不完整"},{"value":"REPEAT_OUTPUT","label":"重复输出"},{"value":"INCONSISTENT_LANGUAGE","label":"语言不一致"},{"value":"FORMAT_ERROR","label":"格式错误"},{"value":"CODE_SYNTAX_ERROR","label":"代码语法错误"},{"value":"IMAGE_COMPREHENSION_ERROR","label":"图片理解错误"},{"value":"LOW_QUALITY_IMAGE_GENERATION","label":"图片生成质量低"},{"value":"BORING","label":"内容没有帮助/无趣"},{"value":"CONTENT_IS_TOO_SHORT","label":"内容过短"},{"value":"REDUNDANCY","label":"内容过长/冗余"}]},{"label":"安全","value":"SAFETY","child":[{"value":"HARMFUL_CONTENT","label":"内容有害（违法违规）"},{"value":"ETHICS","label":"价值观（伦理道德）"},{"value":"PRIVACY_LEAKAGE","label":"隐私泄露"},{"value":"EXCESSIVE_REFUSAL_TO_ANSWER","label":"过度拒答"}]},{"label":"产品","value":"PRODUCT","child":[{"value":"NO_RESPONSE","label":"系统BUG/无响应"},{"value":"SLOW_OPERATION","label":"运行慢/体验差"}]}]',1,null,null,'CONFIG','admin'); + +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(10,now(),now(),'BaiLing','BaiLing','1','SINGLE','chat','/img/logo/bailing.png','LLM','[]','system','system'); +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(20,now(),now(),'BaiLian','BaiLian','1','SINGLE','chat,embedding','/img/logo/bailian.png','LLM,TEXT EMBEDDING','[]','system','system'); +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(30,now(),now(),'DeepSeek','DeepSeek','1','ALL','chat','/img/logo/deepseek.png','LLM','[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://api.siliconflow.cn/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true}]','system','system'); +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(40,now(),now(),'Ollama','Ollama','1','SINGLE','chat,embedding','/img/logo/ollama.png','LLM,TEXT EMBEDDING','[]','system','system'); +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(50,now(),now(),'SILICONFLOW','SILICONFLOW','1','SINGLE','chat,embedding','/img/logo/siliconflow.png','LLM,TEXT EMBEDDING','[]','system','system'); +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(60,now(),now(),'OpenAI','OpenAI','1','SINGLE','chat,embedding','/img/logo/openai.png','LLM,TEXT EMBEDDING','[]','system','system'); +INSERT INTO kg_model_provider (`id`,`gmt_create`,`gmt_modified`,`name`,`provider`,`status`,`page_mode`,`model_type`,`logo`,`tags`,`params`,`creator`,`modifier`) VALUES(70,now(),now(),'OpenRouter','OpenRouter','1','SINGLE','chat','/img/logo/openrouter.png','LLM','[]','system','system'); + +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(1,now(),now(),'BaiLing','chat',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://antchat.alipay.com/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(2,now(),now(),'BaiLian','chat',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://dashscope.aliyuncs.com/compatible-mode/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(3,now(),now(),'BaiLian','embedding',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://dashscope.aliyuncs.com/compatible-mode/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(4,now(),now(),'ollama','chat',null,'[{"inputType":"select","cname":"Model","ename":"model","required":true},{"inputType":"text","cname":"base_url","ename":"base_url","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(5,now(),now(),'ollama','embedding',null,'[{"inputType":"select","cname":"Model","ename":"model","required":true},{"inputType":"text","cname":"base_url","ename":"base_url","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(6,now(),now(),'SILICONFLOW','chat',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://api.siliconflow.cn/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(7,now(),now(),'SILICONFLOW','embedding',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://api.siliconflow.cn/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(8,now(),now(),'OpenAI','chat',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://api.openai.com"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(9,now(),now(),'OpenAI','embedding',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://api.openai.com"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); +INSERT INTO kg_provider_param (`id`,`gmt_create`,`gmt_modified`,`provider`,`model_type`,`model_name`,`params`,`creator`,`modifier`) VALUES(10,now(),now(),'OpenRouter','chat',null,'[{"inputType":"text","cname":"base_url","ename":"base_url","required":true,"defaultValue":"https://openrouter.ai/api/v1"},{"inputType":"text","cname":"API-Key","ename":"api_key","required":true},{"inputType":"text","cname":"Model","ename":"model","required":true}]','system','system'); + +INSERT INTO kg_model_detail (`id`,`gmt_create`,`gmt_modified`,`provider`,`type`,`name`,`description`,`params`,`creator`,`modifier`) VALUES(1,now(),now(),'DeepSeek','chat','deepseek-chat','deepseek-chat',null,'system','system'); +INSERT INTO kg_model_detail (`id`,`gmt_create`,`gmt_modified`,`provider`,`type`,`name`,`description`,`params`,`creator`,`modifier`) VALUES(2,now(),now(),'DeepSeek','chat','deepseek-reasoner','deepseek-reasoner',null,'system','system'); diff --git a/reasoner/catalog/openspg-catalog/src/main/scala/com/antgroup/openspg/reasoner/catalog/impl/PropertySchemaOps.scala b/reasoner/catalog/openspg-catalog/src/main/scala/com/antgroup/openspg/reasoner/catalog/impl/PropertySchemaOps.scala index 11ba7760b..07fdaf0aa 100644 --- a/reasoner/catalog/openspg-catalog/src/main/scala/com/antgroup/openspg/reasoner/catalog/impl/PropertySchemaOps.scala +++ b/reasoner/catalog/openspg-catalog/src/main/scala/com/antgroup/openspg/reasoner/catalog/impl/PropertySchemaOps.scala @@ -49,6 +49,8 @@ object PropertySchemaOps { spgType match { case entityType: EntityType => KTAdvanced(entityType.getName) + case indexType: IndexType => + KTAdvanced(indexType.getName) case conceptType: ConceptType => KTConcept(conceptType.getName) case eventType: EventType => @@ -66,6 +68,8 @@ object PropertySchemaOps { spgType match { case SPGTypeEnum.ENTITY_TYPE => NodeType.ADVANCED + case SPGTypeEnum.INDEX_TYPE => + NodeType.ADVANCED case SPGTypeEnum.CONCEPT_TYPE => NodeType.CONCEPT case SPGTypeEnum.STANDARD_TYPE => diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/ApiConstants.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/ApiConstants.java index 42db1bd76..92f9a0653 100644 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/ApiConstants.java +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/ApiConstants.java @@ -19,14 +19,4 @@ public class ApiConstants { public static final String REMOTE = "Remote"; public static final String TOKEN = "token"; - - public static final String ENV = "env"; - public static final String ACCESS_ID = "accessId"; - public static final String ACCESS_KEY = "accessKey"; - public static final String HOST = "host"; - public static final String TIMEOUT = "timeout"; - public static final String PORT = "port"; - public static final String SCHEME = "scheme"; - - public static final Long DEFAULT_VERSION = 0L; } diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/AppApiKeyRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/AppApiKeyRequest.java new file mode 100644 index 000000000..92d4c8c45 --- /dev/null +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/AppApiKeyRequest.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.api.facade.dto.common.request; + +import lombok.Data; + +@Data +public class AppApiKeyRequest { + private String apiKey; + private String name; + private Integer status; +} diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/AppRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/AppRequest.java new file mode 100644 index 000000000..8db0cdf93 --- /dev/null +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/AppRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.api.facade.dto.common.request; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.common.model.base.BaseRequest; +import java.util.List; +import lombok.Data; + +@Data +public class AppRequest extends BaseRequest { + private Long id; + private String name; + private String logo; + private String description; + private JSONObject config; + private String userNo; + private String alias; + + private List idList; +} diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectCreateRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectCreateRequest.java index 62f304f09..65ffb7d18 100644 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectCreateRequest.java +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectCreateRequest.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.server.api.facade.dto.common.request; +import com.alibaba.fastjson.JSONObject; import com.antgroup.openspg.server.common.model.base.BaseRequest; import lombok.Data; @@ -27,9 +28,17 @@ public class ProjectCreateRequest extends BaseRequest { private String namespace; + private String visibility; + private Long tenantId; - private String config; + private JSONObject config; private Boolean autoSchema; + + private String tag; + + private String userNo; + + private Boolean isKnext; } diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectQueryRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectQueryRequest.java index d21b04316..3bad4b05d 100644 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectQueryRequest.java +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/common/request/ProjectQueryRequest.java @@ -34,10 +34,22 @@ public class ProjectQueryRequest extends BaseRequest { private List projectIdList; + private Boolean isOwner; + + private String visibility; + public Long getTenantId() { return tenantId; } + public Boolean getOwner() { + return isOwner; + } + + public void setOwner(Boolean owner) { + isOwner = owner; + } + public ProjectQueryRequest setTenantId(Long tenantId) { this.tenantId = tenantId; return this; @@ -101,4 +113,12 @@ public String getConfig() { public void setConfig(String config) { this.config = config; } + + public String getVisibility() { + return visibility; + } + + public void setVisibility(String visibility) { + this.visibility = visibility; + } } diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/schema/request/ConceptLevelInstanceRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/schema/request/ConceptLevelInstanceRequest.java index a2f72c86a..9218dc1b1 100644 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/schema/request/ConceptLevelInstanceRequest.java +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/schema/request/ConceptLevelInstanceRequest.java @@ -18,6 +18,7 @@ @Data public class ConceptLevelInstanceRequest { + private Long projectId; private String conceptType; private String rootConceptInstance = ConceptIdentifier.ROOT; } diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/service/request/CustomSearchRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/service/request/CustomSearchRequest.java new file mode 100644 index 000000000..c85e59a9e --- /dev/null +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/service/request/CustomSearchRequest.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.api.facade.dto.service.request; + +import lombok.Data; + +@Data +public class CustomSearchRequest { + private Long projectId; + private String customQuery; +} diff --git a/server/api/http-client/pom.xml b/server/api/http-client/pom.xml index 58232efdf..f31d39051 100644 --- a/server/api/http-client/pom.xml +++ b/server/api/http-client/pom.xml @@ -28,6 +28,10 @@ com.antgroup.openspg.server api-facade + + com.antgroup.openspg + common-util + com.dtflys.forest forest-core diff --git a/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java b/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java index f6d3ceb7f..aa0d57096 100644 --- a/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java +++ b/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.server.api.http.client.forest; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import com.dtflys.forest.converter.ConvertOptions; import com.dtflys.forest.converter.json.ForestJsonConverter; import com.dtflys.forest.http.ForestRequest; diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java index 8c4eb11b2..e8584840b 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.server.api.http.server; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import java.util.Collections; import java.util.List; import org.springframework.context.annotation.Configuration; diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ConceptInstanceController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ConceptInstanceController.java index fd1cf2189..e9e3d9ae8 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ConceptInstanceController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ConceptInstanceController.java @@ -57,7 +57,9 @@ public ConceptLevelInstanceResponse action() { @RequestMapping(method = RequestMethod.GET) @ResponseBody public ResponseEntity query( - @RequestParam String conceptType, @RequestParam Set conceptInstanceIds) { + @RequestParam(required = false) Long projectId, + @RequestParam String conceptType, + @RequestParam Set conceptInstanceIds) { return HttpBizTemplate.execute( new HttpBizCallback>() { @Override @@ -65,7 +67,7 @@ public void check() {} @Override public List action() { - return conceptInstanceManager.query(conceptType, conceptInstanceIds); + return conceptInstanceManager.query(projectId, conceptType, conceptInstanceIds); } }); } diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/DataSourceController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/DataSourceController.java index c126ff65d..bd72ce76f 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/DataSourceController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/DataSourceController.java @@ -62,7 +62,7 @@ public void check() { public Boolean action() { if (StringUtils.isBlank(request.getCreateUser())) { Account account = getLoginAccount(); - String user = account != null ? account.getAccount() : BuilderConstant.SYSTEM; + String user = account != null ? account.getWorkNo() : BuilderConstant.SYSTEM; request.setCreateUser(user); } request.setUpdateUser(request.getCreateUser()); @@ -88,7 +88,7 @@ public void check() { @Override public Boolean action() { Account account = getLoginAccount(); - String user = account != null ? account.getAccount() : BuilderConstant.SYSTEM; + String user = account != null ? account.getWorkNo() : BuilderConstant.SYSTEM; request.setUpdateUser(user); return dataSourceService.update(request) > 0; } diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java index 47edcccf9..0675c2b8e 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ProjectController.java @@ -13,17 +13,29 @@ package com.antgroup.openspg.server.api.http.server.openapi; +import static com.antgroup.openspg.common.constants.SpgAppConstant.ACCOUNT_PATTERN; + import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.constants.SpgAppConstant; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.common.util.enums.PermissionEnum; +import com.antgroup.openspg.common.util.enums.ProjectTagEnum; +import com.antgroup.openspg.common.util.enums.ResourceTagEnum; +import com.antgroup.openspg.common.util.exception.SpgException; +import com.antgroup.openspg.common.util.exception.message.SpgMessageEnum; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectCreateRequest; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest; import com.antgroup.openspg.server.api.facade.dto.schema.request.SchemaAlterRequest; import com.antgroup.openspg.server.api.http.server.BaseController; import com.antgroup.openspg.server.api.http.server.HttpBizCallback; import com.antgroup.openspg.server.api.http.server.HttpBizTemplate; -import com.antgroup.openspg.server.biz.common.ConfigManager; +import com.antgroup.openspg.server.biz.common.PermissionManager; import com.antgroup.openspg.server.biz.common.ProjectManager; +import com.antgroup.openspg.server.biz.common.util.AssertUtils; +import com.antgroup.openspg.server.common.model.exception.IllegalParamsException; import com.antgroup.openspg.server.common.model.project.Project; +import com.baidu.brpc.utils.CollectionUtils; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.utils.Lists; @@ -44,7 +56,7 @@ public class ProjectController extends BaseController { @Autowired private ProjectManager projectManager; - @Autowired private ConfigManager configManager; + @Autowired private PermissionManager permissionManager; @RequestMapping(method = RequestMethod.POST) public ResponseEntity create(@RequestBody ProjectCreateRequest request) { @@ -52,11 +64,36 @@ public ResponseEntity create(@RequestBody ProjectCreateRequest request) return HttpBizTemplate.execute( new HttpBizCallback() { @Override - public void check() {} + public void check() { + AssertUtils.assertParamObjectIsNotNull("request body", request); + AssertUtils.assertParamObjectIsNotNull("userNo", request.getUserNo()); + if (!ACCOUNT_PATTERN.matcher(request.getUserNo()).matches()) { + throw new IllegalParamsException( + "account(userNo) length is 6-20, only support letters,numbers and underscores"); + } + AssertUtils.assertParamObjectIsNotNull("name", request.getName()); + AssertUtils.assertParamObjectIsNotNull("namespace", request.getNamespace()); + AssertUtils.assertParamStringIsNotBlank("tag", request.getTag()); + AssertUtils.assertParamStringIsNotBlank("visibility", request.getVisibility()); + AssertUtils.assertParamObjectIsNotNull("config", request.getConfig()); + AssertUtils.assertParamIsTrue( + "namespace length >= 3", request.getNamespace().length() >= 3); + if (StringUtils.equals(request.getTag(), ProjectTagEnum.LOCAL.name()) + && !request.getConfig().containsKey(SpgAppConstant.VECTORIZER)) { + throw new IllegalParamsException("tag = LOCAL, vectorizer cannot be empty"); + } + } @Override public Project action() { + request.setIsKnext(Boolean.TRUE); Project project = projectManager.create(request); + if (null == project || null == project.getId()) { + return null; + } + if (ProjectTagEnum.PUBLIC_NET.name().equalsIgnoreCase(request.getTag())) { + return project; + } if (request.getAutoSchema() == null || Boolean.TRUE.equals(request.getAutoSchema())) { SchemaAlterRequest request = new SchemaAlterRequest(); request.setProjectId(project.getId()); @@ -90,7 +127,7 @@ public List action() { String config = project.getConfig(); JSONObject configJson = JSON.parseObject(config); if (configJson != null) { - configManager.backwardCompatible(configJson); + projectManager.completionVectorizer(configJson); config = configJson.toJSONString(); } newProjectList.add( @@ -100,7 +137,8 @@ public List action() { project.getDescription(), project.getNamespace(), project.getTenantId(), - config)); + config, + project.getTag())); }); return newProjectList; } @@ -112,11 +150,37 @@ public ResponseEntity update(@RequestBody ProjectCreateRequest request) log.info("HTTP Update Project Params: {}", JSON.toJSONString(request)); return HttpBizTemplate.execute( new HttpBizCallback() { + @Override - public void check() {} + public void check() { + AssertUtils.assertParamObjectIsNotNull("request body", request); + AssertUtils.assertParamObjectIsNotNull("id", request.getId()); + AssertUtils.assertParamStringIsNotBlank("userNo", request.getUserNo()); + AssertUtils.assertParamObjectIsNotNull("config", request.getConfig()); + String userNo = request.getUserNo(); + boolean hasPermission = + permissionManager.hasPermission( + userNo, + request.getId(), + ResourceTagEnum.KNOWLEDGE_BASE.name(), + PermissionEnum.OWNER.name()); + if (!hasPermission) { + throw new SpgException(SpgMessageEnum.KB_NOT_OWNER); + } + Project project = projectManager.queryById(request.getId()); + AssertUtils.assertParamObjectIsNotNull("query project by id", project); + ProjectQueryRequest projectQueryRequest = new ProjectQueryRequest(); + projectQueryRequest.setName(request.getName()); + List query = projectManager.query(projectQueryRequest); + if (CollectionUtils.isNotEmpty(query) + && query.stream().anyMatch(it -> !it.getId().equals(request.getId()))) { + throw new SpgException(SpgMessageEnum.KB_NAME_EXIST); + } + } @Override public Project action() { + request.setIsKnext(Boolean.TRUE); return projectManager.update(request); } }); diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/RetrievalController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/RetrievalController.java new file mode 100644 index 000000000..d9394dea5 --- /dev/null +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/RetrievalController.java @@ -0,0 +1,151 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.api.http.server.openapi; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.server.api.http.server.BaseController; +import com.antgroup.openspg.server.api.http.server.HttpBizCallback; +import com.antgroup.openspg.server.api.http.server.HttpBizTemplate; +import com.antgroup.openspg.server.api.http.server.HttpResult; +import com.antgroup.openspg.server.biz.common.util.AssertUtils; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.common.model.retrieval.RetrievalQuery; +import com.antgroup.openspg.server.common.service.builder.BuilderJobService; +import com.antgroup.openspg.server.common.service.retrieval.RetrievalService; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@RequestMapping("/public/v1/retrieval") +@Slf4j +public class RetrievalController extends BaseController { + + @Autowired private RetrievalService retrievalService; + + @Autowired private BuilderJobService builderJobService; + + @RequestMapping(value = "/getAll", method = RequestMethod.GET) + @ResponseBody + public HttpResult> getAll() { + return HttpBizTemplate.execute2( + new HttpBizCallback>() { + @Override + public void check() { + log.info("/retrieval/getAll"); + } + + @Override + public List action() { + return retrievalService.query(new RetrievalQuery()).getResults(); + } + }); + } + + @RequestMapping(value = "/delete", method = RequestMethod.GET) + @ResponseBody + public HttpResult delete(Long id) { + return HttpBizTemplate.execute2( + new HttpBizCallback() { + @Override + public void check() { + log.info("/retrieval/delete id: {}", id); + AssertUtils.assertParamObjectIsNotNull("id", id); + } + + @Override + public Boolean action() { + return retrievalService.deleteById(id) > 0; + } + }); + } + + @RequestMapping(value = "/getById", method = RequestMethod.GET) + @ResponseBody + public HttpResult getById(Long id) { + return HttpBizTemplate.execute2( + new HttpBizCallback() { + @Override + public void check() { + log.info("/retrieval/getById id: {}", id); + AssertUtils.assertParamObjectIsNotNull("id", id); + } + + @Override + public Retrieval action() { + return retrievalService.getById(id); + } + }); + } + + @RequestMapping(value = "/getByProjectId", method = RequestMethod.GET) + @ResponseBody + public HttpResult> getByProjectId(Long projectId) { + return HttpBizTemplate.execute2( + new HttpBizCallback>() { + @Override + public void check() { + log.info("/retrieval/getByProjectId projectId: {}", projectId); + AssertUtils.assertParamObjectIsNotNull("projectId", projectId); + } + + @Override + public List action() { + return retrievalService.getRetrievalByProjectId(projectId); + } + }); + } + + @RequestMapping(value = "/search", method = RequestMethod.POST) + @ResponseBody + public HttpResult> search(@RequestBody RetrievalQuery request) { + return HttpBizTemplate.execute2( + new HttpBizCallback>() { + @Override + public void check() { + log.info("/retrieval/search request: {}", JSON.toJSONString(request)); + } + + @Override + public Paged action() { + return retrievalService.query(request); + } + }); + } + + @RequestMapping(value = "/update", method = RequestMethod.POST) + @ResponseBody + public HttpResult update(@RequestBody Retrieval request) { + return HttpBizTemplate.execute2( + new HttpBizCallback() { + @Override + public void check() { + log.info("/retrieval/update request: {}", JSON.toJSONString(request)); + AssertUtils.assertParamObjectIsNotNull("retrieval", request); + AssertUtils.assertParamObjectIsNotNull("id", request.getId()); + } + + @Override + public Long action() { + return retrievalService.update(request); + } + }); + } +} diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java index 9cdbf7133..10b815ad1 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SchemaController.java @@ -73,7 +73,7 @@ public static List getDefaultSchema(String namespace) { chunkProperties.put("content", IndexTypeEnum.TEXT_AND_VECTOR); Map properties = Maps.newHashMap(); - properties.put("desc", IndexTypeEnum.TEXT_AND_VECTOR); + // properties.put("desc", IndexTypeEnum.TEXT_AND_VECTOR); properties.put("semanticType", IndexTypeEnum.TEXT); schemaTypes.add(getBaseSPGType(namespace, "Chunk", "文本块", chunkProperties)); diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SearchController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SearchController.java index a373782e9..214f60eb8 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SearchController.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/SearchController.java @@ -14,6 +14,7 @@ package com.antgroup.openspg.server.api.http.server.openapi; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.idx.record.IdxRecord; +import com.antgroup.openspg.server.api.facade.dto.service.request.CustomSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.SPGTypeSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.TextSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.VectorSearchRequest; @@ -76,4 +77,18 @@ public List action() { } }); } + + @RequestMapping(method = RequestMethod.POST, value = "/custom") + public ResponseEntity textSearch(@RequestBody CustomSearchRequest request) { + return HttpBizTemplate.execute( + new HttpBizCallback>() { + @Override + public void check() {} + + @Override + public List action() { + return searchManager.customSearch(request); + } + }); + } } diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/AppManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/AppManager.java new file mode 100644 index 000000000..3a22a26d7 --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/AppManager.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.biz.common; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.api.facade.dto.common.request.AppRequest; +import com.antgroup.openspg.server.common.model.app.App; +import java.util.List; +import java.util.Map; + +public interface AppManager { + Long create(AppRequest request); + + Integer update(AppRequest request); + + Integer deleteById(Long id); + + App queryById(Long id); + + List queryPage(AppRequest request, Long page, Long size); + + Long selectCountByCondition(AppRequest request); + + Map getKbIdsAndLLMIdByConfig(JSONObject config); + + App queryByName(String name); + + List queryByAlias(String alias); +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ModelDetailManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ModelDetailManager.java new file mode 100644 index 000000000..d615b604c --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ModelDetailManager.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common; + +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetail; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import java.util.List; + +public interface ModelDetailManager { + + /** insert model provider */ + Long insert(ModelDetail record); + + /** delete by id */ + int deleteById(Long id); + + /** update model provider */ + Long update(ModelDetail record); + + /** get by id */ + ModelDetail getById(Long id); + + /** query by condition */ + List query(ModelDetailQuery record); + + /** + * query dto by condition + * + * @param modelDetailQuery + * @return + */ + List queryDTO(ModelDetailQuery modelDetailQuery); +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ModelProviderManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ModelProviderManager.java new file mode 100644 index 000000000..bab5be3ea --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ModelProviderManager.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common; + +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import java.util.List; + +public interface ModelProviderManager { + + /** insert model provider */ + Long insert(ModelProvider record); + + /** delete by id */ + int deleteById(Long id); + + /** update model provider */ + Long update(ModelProvider record); + + /** get by id */ + ModelProvider getById(Long id); + + /** get by provider */ + ModelProvider getByProvider(String provider); + + /** query by condition */ + List query(String modelType); + + /** get by provider list */ + List selectByProviders(List providerList); +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/PermissionManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/PermissionManager.java index f83ac088d..f350910e1 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/PermissionManager.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/PermissionManager.java @@ -13,7 +13,6 @@ package com.antgroup.openspg.server.biz.common; -import com.antgroup.openspg.server.api.facade.Paged; import com.antgroup.openspg.server.api.facade.dto.common.request.PermissionRequest; import com.antgroup.openspg.server.common.model.permission.Permission; import java.util.List; @@ -37,25 +36,6 @@ public interface PermissionManager { */ Integer update(PermissionRequest request); - /** - * query page permission or query by condition - * - * @param userNo - * @param roleType - * @param resourceId - * @param resourceTag - * @param page - * @param size - * @return - */ - Paged query( - String userNo, - String roleType, - Long resourceId, - String resourceTag, - Integer page, - Integer size); - /** * remove permission * @@ -88,21 +68,34 @@ List getPermissionByUserRolesAndId( boolean isSuper(String userNo); /** - * is project role + * has permission * * @param userNo - * @param projectId + * @param resourceId + * @param resourceType + * @param roleType * @return */ - boolean isProjectRole(String userNo, Long projectId); + boolean hasPermission(String userNo, Long resourceId, String resourceType, String roleType); /** - * get owner user name by project id + * is role members * - * @param projectId + * @param userNo + * @param resourceId + * @param resourceType * @return */ - List getOwnerUserNameByProjectId(Long projectId); + boolean isRoleMembers(String userNo, Long resourceId, String resourceType); + + /** + * get owner user name by resource id + * + * @param resourceId + * @param resourceType + * @return + */ + List getOwnerUserNameByResourceId(Long resourceId, String resourceType); /** * get by id @@ -111,4 +104,39 @@ List getPermissionByUserRolesAndId( * @return */ Permission selectByPrimaryKey(Long id); + + /** + * select page + * + * @param userNo + * @param roleType + * @param resourceId + * @param resourceTag + * @param start + * @param size + * @return + */ + List selectLikeByUserNoAndRoleId( + String userNo, String roleType, Long resourceId, String resourceTag, Long start, Long size); + + /** + * the count of selectLikeByUserNoAndRoleId + * + * @param userNo + * @param roleType + * @param resourceId + * @param resourceTag + * @return + */ + long selectLikeCountByUserNoAndRoleId( + String userNo, String roleType, Long resourceId, String resourceTag); + + /** + * Delete Records Corresponding to the Resource + * + * @param resourceId + * @param resourceTag + * @return + */ + int deleteByResourceId(Long resourceId, String resourceTag); } diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProjectManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProjectManager.java index 0945c5a1b..2b48f141c 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProjectManager.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProjectManager.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.server.biz.common; -import com.antgroup.openspg.server.api.facade.Paged; +import com.alibaba.fastjson.JSONObject; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectCreateRequest; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest; import com.antgroup.openspg.server.common.model.project.Project; @@ -31,8 +31,9 @@ public interface ProjectManager { List query(ProjectQueryRequest request); - Paged queryPaged(ProjectQueryRequest request, int start, int size); + List queryPageData(ProjectQueryRequest request, int start, int size); + Long queryPageCount(ProjectQueryRequest request); /** * get GraphStore Url by project ID. * @@ -56,4 +57,11 @@ public interface ProjectManager { * @return */ Project queryByNamespace(String namespace); + + /** + * completion vectorizer + * + * @param projectConfig + */ + void completionVectorizer(JSONObject projectConfig); } diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProviderParamManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProviderParamManager.java new file mode 100644 index 000000000..198d2d62c --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ProviderParamManager.java @@ -0,0 +1,37 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common; + +import com.antgroup.openspg.server.common.model.providerparam.ProviderParam; +import java.util.List; + +public interface ProviderParamManager { + + /** insert provider param */ + Long insert(ProviderParam record); + + /** delete by id */ + int deleteById(Long id); + + /** update provider param */ + Long update(ProviderParam record); + + /** get by id */ + ProviderParam getById(Long id); + + /** query provider param */ + List query(String provider, String modelType); + + /** get by provider and model type */ + ProviderParam getByProviderAndModelType(String provider, String modelType); +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/RefManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/RefManager.java new file mode 100644 index 000000000..3e57422cf --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/RefManager.java @@ -0,0 +1,72 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.biz.common; + +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.common.model.ref.RefTypeEnum; +import com.antgroup.openspg.server.common.model.ref.RefedTypeEnum; +import java.util.List; + +public interface RefManager { + + /** + * create a new reference + * + * @param refInfo + * @return + */ + Long create(RefInfo refInfo); + + /** Get a list of RefInfo objects by refId and refType. */ + List getRefInfoByRef(String refId, RefTypeEnum refType); + + /** Get a list of RefInfo objects by refedId and refedType. */ + List getRefInfoByRefed(String refedId, RefedTypeEnum refedType); + + /** + * Get a list of RefInfo objects by right matching (suffix) the refedId. + * + * @param refedId The string to use for right matching + * @return A list of matching RefInfo objects + */ + List getRefInfoByRightMatchRefedId(String refedId); + + /** select by unique key */ + RefInfo getByUniqueKey( + String refId, RefTypeEnum refType, String refedId, RefedTypeEnum refedType); + + /** update by primary key */ + int updateByPrimaryKeySelective(RefInfo refInfo); + + /** update by unique key */ + int updateByUniqueKey(RefInfo refInfo); + + /** delete by primary key */ + int deleteById(Long id); + + /** delete by multiple primary keys */ + int deleteByIds(List ids); + + /** delete by unique key */ + int deleteByUniqueKey(RefInfo refInfo); + + /** + * Record the usage information of an API key. + * + * @param appId The ID of the application using the API key + * @param apiKey The API key being used + * @param uri The URI of the API endpoint being accessed + */ + void recordApiKeyUsageInfo(String appId, String apiKey, String uri); +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/UserModelManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/UserModelManager.java new file mode 100644 index 000000000..df966d811 --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/UserModelManager.java @@ -0,0 +1,154 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.common.model.usermodel.UserModel; +import com.antgroup.openspg.server.common.model.usermodel.UserModelDTO; +import com.antgroup.openspg.server.common.model.usermodel.UserModelQuery; +import java.util.List; +import java.util.Map; + +public interface UserModelManager { + + /** insert user model */ + Long insert(UserModelDTO record, Map modelTypeMap); + + /** delete By Id */ + int deleteById(Long id); + + /** delete By Ids */ + int deleteByIds(List ids); + + /** update user model */ + Long update(UserModel record); + + /** get By id */ + UserModel getById(Long id); + + /** query By Condition */ + List query(UserModelQuery record); + + /** get By Provider And Name */ + List getByProviderAndName(String provider, String name); + + /** + * update api key + * + * @param request + * @return + */ + Long updateApiKey(UserModelDTO request); + + /** + * list + * + * @param modelType + * @param queryStr + * @param modelId + * @param userNo + * @return + */ + List> list(String modelType, String queryStr, String modelId, String userNo); + + /** + * get by model unique id + * + * @param modelId + * @return + */ + JSONObject getByModelId(String modelId); + + /** + * delete model + * + * @param id + * @param modelId + * @return + */ + Boolean deleteModel(Long id, String modelId); + + /** + * update model visibility + * + * @param modelId + * @param visibility + * @param customize + * @return + */ + Boolean updateModelVisibility(String modelId, String visibility, JSONObject customize); + + /** + * get by instance id + * + * @param instanceId + * @return + */ + UserModel getByInstanceId(String instanceId); + + /** + * update base info by ids + * + * @param ids + * @param name + * @param visibility + * @param userNo + * @param config + * @return + */ + Long updateBaseInfoByIds( + List ids, String name, String visibility, String userNo, String config); + + /** + * select user private or public + * + * @param userNo + * @return + */ + List selectUserPrivateOrPublic(String userNo); + + /** + * desensitized field sensitive + * + * @param config + * @return + */ + JSONObject desensitizedFieldSensitive(JSONObject config); + + /** + * sensitive field recovery + * + * @param config + * @return + */ + JSONObject sensitiveFieldRecovery(JSONObject config, JSONObject oldConfig); + + /** + * get model type map + * + * @return + */ + Map getModelTypeMap(); + + /** + * get model by name and model + * + * @param provider + * @param name + * @param visibility + * @param model + * @return + */ + JSONObject getModelByProviderAndModel( + String provider, String name, String visibility, String model); +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/AppManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/AppManagerImpl.java new file mode 100644 index 000000000..9519f7b3e --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/AppManagerImpl.java @@ -0,0 +1,218 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.constants.SpgAppConstant; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.common.util.enums.PermissionEnum; +import com.antgroup.openspg.common.util.enums.ResourceTagEnum; +import com.antgroup.openspg.server.api.facade.dto.common.request.AppRequest; +import com.antgroup.openspg.server.api.facade.dto.common.request.PermissionRequest; +import com.antgroup.openspg.server.biz.common.AppManager; +import com.antgroup.openspg.server.biz.common.PermissionManager; +import com.antgroup.openspg.server.biz.common.RefManager; +import com.antgroup.openspg.server.common.model.app.App; +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.common.model.ref.RefTypeEnum; +import com.antgroup.openspg.server.common.model.ref.RefedTypeEnum; +import com.antgroup.openspg.server.common.service.app.AppRepository; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Slf4j +@Service +public class AppManagerImpl implements AppManager { + + @Autowired private AppRepository appRepository; + @Autowired private PermissionManager permissionManager; + @Autowired private RefManager refManager; + + @Override + public Long create(AppRequest request) { + App app = + new App( + null, + request.getName(), + request.getLogo(), + request.getDescription(), + request.getConfig(), + request.getUserNo(), + request.getAlias()); + appRepository.save(app); + if (app != null && app.getId() != null) { + PermissionRequest permissionRequest = new PermissionRequest(); + permissionRequest.setResourceIds(Lists.newArrayList(app.getId())); + permissionRequest.setResourceTag(ResourceTagEnum.APP.name()); + permissionRequest.setUserNos(Lists.newArrayList(request.getUserNo())); + permissionRequest.setRoleType(PermissionEnum.OWNER.name()); + permissionManager.create(permissionRequest); + } + return app.getId(); + } + + @Override + @Transactional(value = "transactionManager", rollbackFor = Exception.class) + public Integer update(AppRequest request) { + App app = + new App( + request.getId(), + request.getName(), + request.getLogo(), + request.getDescription(), + request.getConfig(), + request.getUserNo(), + request.getAlias()); + Integer update = appRepository.update(app); + + if (request.getConfig() == null) { + return update; + } + Map idMap = getKbIdsAndLLMIdByConfig(request.getConfig()); + List kbIds = (List) idMap.get("kbIds"); + String llmId = (String) idMap.get("llmId"); + Map kbId2StatusMap = (Map) idMap.get("kbId2StatusMap"); + List refInfoByRef = + refManager.getRefInfoByRef(String.valueOf(request.getId()), RefTypeEnum.APP); + if (CollectionUtils.isNotEmpty(refInfoByRef)) { + List ids = + refInfoByRef.stream() + .filter( + ref -> + Arrays.asList(RefTypeEnum.KNOWLEDGE_BASE.name(), RefedTypeEnum.LLM.name()) + .contains(ref.getRefedType())) + .map(RefInfo::getId) + .collect(Collectors.toList()); + // Delete the association between the knowledge base or llm and the application + refManager.deleteByIds(ids); + } + + if (CollectionUtils.isNotEmpty(kbIds)) { + for (Long kbId : kbIds) { + RefInfo refInfo = + new RefInfo( + "APP_KNOWLEDGE_BASE", + String.valueOf(request.getId()), + RefTypeEnum.APP.name(), + String.valueOf(kbId), + RefTypeEnum.KNOWLEDGE_BASE.name(), + Boolean.TRUE.equals(kbId2StatusMap.get(kbId)) ? 1 : 0); + refManager.create(refInfo); + log.info( + "create app knowledge base ref info, refId: {}, refInfo: {}", + kbId, + JSON.toJSONString(refInfo)); + } + } + if (StringUtils.isNotBlank(llmId)) { + RefInfo refInfo = + new RefInfo( + "APP_LLM", + String.valueOf(request.getId()), + RefTypeEnum.APP.name(), + llmId, + RefedTypeEnum.LLM.name(), + 1); + Long id = refManager.create(refInfo); + log.info("create app llm ref info, refId: {}, refInfo: {}", id, JSON.toJSONString(refInfo)); + } + return update; + } + + @Override + @Transactional(value = "transactionManager", rollbackFor = Exception.class) + public Integer deleteById(Long id) { + Integer delete = appRepository.deleteById(id); + permissionManager.deleteByResourceId(id, ResourceTagEnum.APP.name()); + List refInfoByRef = refManager.getRefInfoByRef(String.valueOf(id), RefTypeEnum.APP); + if (CollectionUtils.isNotEmpty(refInfoByRef)) { + List ids = refInfoByRef.stream().map(RefInfo::getId).collect(Collectors.toList()); + // Delete the association between the knowledge base or llm and the application + refManager.deleteByIds(ids); + } + return delete; + } + + @Override + public App queryById(Long id) { + App app = appRepository.queryById(id); + if (app == null) { + return null; + } + return app; + } + + @Override + public List queryPage(AppRequest request, Long page, Long size) { + return appRepository.queryPage(request, page.intValue(), size.intValue()); + } + + @Override + public Long selectCountByCondition(AppRequest request) { + return appRepository.selectCountByCondition(request); + } + + @Override + public Map getKbIdsAndLLMIdByConfig(JSONObject config) { + Map map = new HashMap<>(2); + Map kbId2StatusMap = new HashMap<>(); + map.put("kbIds", Lists.newArrayList()); + map.put("llmId", null); + map.put("kbId2StatusMap", kbId2StatusMap); + if (config == null) { + return map; + } + JSONArray kbArray = config.getJSONArray("kb"); + if (CollectionUtils.isNotEmpty(kbArray)) { + List kbIds = Lists.newArrayList(); + for (int i = 0; i < kbArray.size(); i++) { + JSONObject jsonObject = kbArray.getJSONObject(i); + Long id = jsonObject.getLong("id"); + kbId2StatusMap.put(id, jsonObject.getBooleanValue("enable")); + kbIds.add(id); + } + map.put("kbIds", kbIds); + } + JSONObject llm = config.getJSONObject(SpgAppConstant.LLM); + if (llm != null) { + String modelId = llm.getString(SpgAppConstant.MODEL_ID); + if (StringUtils.isNotBlank(modelId)) { + map.put("llmId", modelId); + } + } + return map; + } + + @Override + public App queryByName(String name) { + return appRepository.queryByName(name); + } + + @Override + public List queryByAlias(String alias) { + AppRequest appRequest = new AppRequest(); + appRequest.setAlias(alias); + return appRepository.queryByCondition(appRequest); + } +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ModelDetailManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ModelDetailManagerImpl.java new file mode 100644 index 000000000..659402a5f --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ModelDetailManagerImpl.java @@ -0,0 +1,58 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common.impl; + +import com.antgroup.openspg.server.biz.common.ModelDetailManager; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetail; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import com.antgroup.openspg.server.common.service.modeldetail.ModelDetailRepository; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ModelDetailManagerImpl implements ModelDetailManager { + + @Autowired private ModelDetailRepository modelDetailRepository; + + @Override + public Long insert(ModelDetail record) { + return modelDetailRepository.insert(record); + } + + @Override + public int deleteById(Long id) { + return modelDetailRepository.deleteById(id); + } + + @Override + public Long update(ModelDetail record) { + return modelDetailRepository.update(record); + } + + @Override + public ModelDetail getById(Long id) { + return modelDetailRepository.getById(id); + } + + @Override + public List query(ModelDetailQuery record) { + return modelDetailRepository.query(record); + } + + @Override + public List queryDTO(ModelDetailQuery modelDetailQuery) { + return modelDetailRepository.queryDTO(modelDetailQuery); + } +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ModelProviderManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ModelProviderManagerImpl.java new file mode 100644 index 000000000..672cf0aa6 --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ModelProviderManagerImpl.java @@ -0,0 +1,66 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common.impl; + +import com.antgroup.openspg.server.biz.common.ModelProviderManager; +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import com.antgroup.openspg.server.common.model.provider.ModelProviderQuery; +import com.antgroup.openspg.server.common.service.provider.ModelProviderRepository; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ModelProviderManagerImpl implements ModelProviderManager { + + @Autowired private ModelProviderRepository modelProviderRepository; + + @Override + public Long insert(ModelProvider record) { + return modelProviderRepository.insert(record); + } + + @Override + public int deleteById(Long id) { + return modelProviderRepository.deleteById(id); + } + + @Override + public Long update(ModelProvider record) { + return modelProviderRepository.update(record); + } + + @Override + public ModelProvider getById(Long id) { + return modelProviderRepository.getById(id); + } + + @Override + public ModelProvider getByProvider(String provider) { + return modelProviderRepository.getByProvider(provider); + } + + @Override + public List query(String modelType) { + ModelProviderQuery record = new ModelProviderQuery(); + record.setModelType(modelType); + record.setOrder("asc"); + record.setSort("id"); + return modelProviderRepository.query(record); + } + + @Override + public List selectByProviders(List providerList) { + return modelProviderRepository.selectByProviders(providerList); + } +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/PermissionManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/PermissionManagerImpl.java index 84a28ea1b..4913f7c82 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/PermissionManagerImpl.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/PermissionManagerImpl.java @@ -16,7 +16,6 @@ import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.common.util.enums.PermissionEnum; import com.antgroup.openspg.common.util.enums.ResourceTagEnum; -import com.antgroup.openspg.server.api.facade.Paged; import com.antgroup.openspg.server.api.facade.dto.common.request.PermissionRequest; import com.antgroup.openspg.server.biz.common.AccountManager; import com.antgroup.openspg.server.biz.common.PermissionManager; @@ -81,47 +80,6 @@ public Integer update(PermissionRequest request) { return count; } - @Override - public Paged query( - String userNo, - String roleType, - Long resourceId, - String resourceTag, - Integer page, - Integer size) { - if (StringUtils.isBlank(resourceTag) || null == resourceId) { - return new Paged<>(); - } - // 翻译roleId - Long roleId = null; - if (StringUtils.isNotBlank(roleType)) { - roleId = PermissionEnum.valueOf(roleType).getId(); - } - Paged permissionPage = - permissionRepository.queryPage(userNo, roleId, resourceId, resourceTag, page, size); - if (permissionPage.getTotal() > 0 && CollectionUtils.isNotEmpty(permissionPage.getResults())) { - List permissionList = permissionPage.getResults(); - Set userNos = - permissionList.stream().map(Permission::getUserNo).collect(Collectors.toSet()); - Map userNameMap = - accountManager.getSimpleAccountByUserNoList(userNos).stream() - .collect(Collectors.toMap(Account::getWorkNo, account -> account)); - permissionList.forEach( - permission -> { - Account account = userNameMap.get(permission.getUserNo()); - if (null != account) { - permission.setUserName( - StringUtils.isNotBlank(account.getNickName()) - ? account.getNickName() - : account.getWorkNo()); - } - permission.setAccountRoleInfo( - new AccountRoleInfo(PermissionEnum.getRoleTypeById(permission.getRoleId()).name())); - }); - } - return permissionPage; - } - @Override public Integer removePermission(PermissionRequest request) { int count = 0; @@ -160,10 +118,24 @@ public boolean isSuper(String userNo) { } @Override - public boolean isProjectRole(String userNo, Long projectId) { + public boolean hasPermission( + String userNo, Long resourceId, String resourceType, String roleType) { + if (isSuper(userNo)) { + return true; + } + if (PermissionEnum.MEMBER.name().equals(roleType)) { + return isRoleMembers(userNo, resourceId, resourceType); + } List permissionList = getPermissionByUserRolesAndId( - Lists.newArrayList(projectId), null, null, ResourceTagEnum.PROJECT.name()); + Lists.newArrayList(resourceId), userNo, roleType, resourceType); + return CollectionUtils.isNotEmpty(permissionList); + } + + @Override + public boolean isRoleMembers(String userNo, Long resourceId, String resourceType) { + List permissionList = + getPermissionByUserRolesAndId(Lists.newArrayList(resourceId), null, null, resourceType); if (CollectionUtils.isEmpty(permissionList)) { return false; } @@ -172,14 +144,11 @@ public boolean isProjectRole(String userNo, Long projectId) { } @Override - public List getOwnerUserNameByProjectId(Long projectId) { + public List getOwnerUserNameByResourceId(Long resourceId, String resourceType) { String defaultOwner = "there is no resource administrator for the project"; List permissionList = getPermissionByUserRolesAndId( - Lists.newArrayList(projectId), - null, - PermissionEnum.OWNER.name(), - ResourceTagEnum.PROJECT.name()); + Lists.newArrayList(resourceId), null, PermissionEnum.OWNER.name(), resourceType); List userNoList = permissionList.stream() .filter(permission -> permission.getRoleId().equals(PermissionEnum.OWNER.getId())) @@ -207,9 +176,57 @@ public Permission selectByPrimaryKey(Long id) { return permissionRepository.selectByPrimaryKey(id); } + @Override + public List selectLikeByUserNoAndRoleId( + String userNo, String roleType, Long resourceId, String resourceTag, Long page, Long size) { + Long roleId = null; + if (StringUtils.isNotBlank(roleType)) { + roleId = PermissionEnum.valueOf(roleType).getId(); + } + List permissionList = + permissionRepository.selectLikeByUserNoAndRoleId( + userNo, roleId, resourceId, resourceTag, page, size); + if (CollectionUtils.isNotEmpty(permissionList)) { + Set userNos = + permissionList.stream().map(Permission::getUserNo).collect(Collectors.toSet()); + Map userNameMap = + accountManager.getSimpleAccountByUserNoList(userNos).stream() + .collect(Collectors.toMap(Account::getWorkNo, account -> account)); + permissionList.forEach( + permission -> { + Account account = userNameMap.get(permission.getUserNo()); + if (null != account) { + permission.setUserName( + StringUtils.isNotBlank(account.getNickName()) + ? account.getNickName() + : account.getWorkNo()); + } + permission.setAccountRoleInfo( + new AccountRoleInfo(PermissionEnum.getRoleTypeById(permission.getRoleId()).name())); + }); + } + return permissionList; + } + + @Override + public long selectLikeCountByUserNoAndRoleId( + String userNo, String roleType, Long resourceId, String resourceTag) { + Long roleId = null; + if (StringUtils.isNotBlank(roleType)) { + roleId = PermissionEnum.valueOf(roleType).getId(); + } + return permissionRepository.selectLikeCountByUserNoAndRoleId( + userNo, roleId, resourceId, resourceTag); + } + + @Override + public int deleteByResourceId(Long resourceId, String resourceTag) { + return permissionRepository.deleteByResourceId(resourceId, resourceTag); + } + private List toModels(PermissionRequest request) { Long roleId = null; - if (StringUtils.equals(request.getResourceTag(), ResourceTagEnum.PROJECT.name())) { + if (StringUtils.isNotBlank(request.getRoleType())) { roleId = PermissionEnum.valueOf(request.getRoleType()).getId(); } List permissionList = new ArrayList<>(); diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java index 17603280a..704824b89 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProjectManagerImpl.java @@ -13,32 +13,56 @@ package com.antgroup.openspg.server.biz.common.impl; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.antgroup.openspg.cloudext.impl.graphstore.neo4j.Neo4jConstants; +import com.antgroup.openspg.common.constants.BuilderConstant; +import com.antgroup.openspg.common.constants.SpgAppConstant; import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.common.util.constants.CommonConstant; +import com.antgroup.openspg.common.util.enums.PermissionEnum; +import com.antgroup.openspg.common.util.enums.ProjectTagEnum; +import com.antgroup.openspg.common.util.enums.ResourceTagEnum; +import com.antgroup.openspg.common.util.enums.VisibilityEnum; import com.antgroup.openspg.common.util.neo4j.Neo4jAdminUtils; +import com.antgroup.openspg.common.util.pemja.PemjaUtils; +import com.antgroup.openspg.common.util.pemja.PythonInvokeMethod; +import com.antgroup.openspg.common.util.pemja.model.PemjaConfig; import com.antgroup.openspg.core.schema.model.SPGSchema; import com.antgroup.openspg.core.schema.model.SPGSchemaAlterCmd; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.type.BaseSPGType; -import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.server.api.facade.dto.common.request.PermissionRequest; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectCreateRequest; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest; +import com.antgroup.openspg.server.biz.common.PermissionManager; import com.antgroup.openspg.server.biz.common.ProjectManager; +import com.antgroup.openspg.server.biz.common.RefManager; +import com.antgroup.openspg.server.biz.common.UserModelManager; +import com.antgroup.openspg.server.biz.common.util.AssertUtils; import com.antgroup.openspg.server.common.model.CommonConstants; +import com.antgroup.openspg.server.common.model.exception.IllegalParamsException; import com.antgroup.openspg.server.common.model.project.Project; +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.common.model.ref.RefTypeEnum; +import com.antgroup.openspg.server.common.model.ref.RefedTypeEnum; +import com.antgroup.openspg.server.common.model.usermodel.UserModelDTO; +import com.antgroup.openspg.server.common.service.config.DefaultValue; import com.antgroup.openspg.server.common.service.project.ProjectRepository; import com.antgroup.openspg.server.common.service.project.ProjectService; import com.antgroup.openspg.server.core.schema.service.alter.sync.BaseSchemaSyncer; import com.antgroup.openspg.server.core.schema.service.alter.sync.SchemaStorageEnum; import com.antgroup.openspg.server.core.schema.service.alter.sync.SchemaSyncerFactory; import com.antgroup.openspg.server.core.schema.service.type.SPGTypeService; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.util.List; +import java.util.Map; import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; @@ -50,13 +74,26 @@ public class ProjectManagerImpl implements ProjectManager { @Autowired private ProjectService projectService; @Autowired private SchemaSyncerFactory schemaSyncerFactory; @Autowired private SPGTypeService spgTypeService; + @Autowired private UserModelManager userModelManager; + @Autowired private PermissionManager permissionManager; + @Autowired private RefManager refManager; + @Autowired private DefaultValue defaultValue; - @Value("${cloudext.graphstore.url:}") - private String url; - + @Transactional(value = "transactionManager", rollbackFor = Exception.class) @Override public Project create(ProjectCreateRequest request) { - JSONObject config = setDatabase(request.getConfig(), request.getNamespace()); + if (!ProjectTagEnum.PUBLIC_NET.name().equalsIgnoreCase(request.getTag())) { + setLocalVectorizer(request); + } + JSONObject config = request.getConfig(); + if (!ProjectTagEnum.PUBLIC_NET.name().equalsIgnoreCase(request.getTag())) { + config = setDatabase(config, request.getNamespace()); + setGraphStore(request.getNamespace(), config, true); + config.remove(SpgAppConstant.MCP_SERVERS); + } else { + config.remove(SpgAppConstant.VECTORIZER); + config.remove(SpgAppConstant.GRAPH_STORE); + } Project project = new Project( null, @@ -64,19 +101,87 @@ public Project create(ProjectCreateRequest request) { request.getDescription(), request.getNamespace(), request.getTenantId(), - config.toJSONString()); - setGraphStore(project, config, true); + config.toJSONString(), + StringUtils.isBlank(request.getTag()) ? ProjectTagEnum.LOCAL.name() : request.getTag()); + if (StringUtils.isNotBlank(request.getVisibility())) { + VisibilityEnum visibilityEnum = VisibilityEnum.getVisibilityEnum(request.getVisibility()); + project.setVisibility(visibilityEnum.name()); + } Long projectId = projectRepository.save(project); project.setId(projectId); + PermissionRequest permissionRequest = new PermissionRequest(); + permissionRequest.setResourceIds(Lists.newArrayList(project.getId())); + permissionRequest.setResourceTag(ResourceTagEnum.KNOWLEDGE_BASE.name()); + permissionRequest.setUserNos(Lists.newArrayList(request.getUserNo())); + permissionRequest.setRoleType(PermissionEnum.OWNER.name()); + permissionManager.create(permissionRequest); + if (ProjectTagEnum.LOCAL.name().equalsIgnoreCase(request.getTag())) { + createRefInfo(project); + } return project; } + private void createRefInfo(Project project) { + JSONObject jsonObject = JSON.parseObject(project.getConfig()); + String embeddingModeId = getEmbeddingModelId(jsonObject); + RefInfo refInfo = + new RefInfo( + "PROJECT_EMBEDDING", + String.valueOf(project.getId()), + RefTypeEnum.KNOWLEDGE_BASE.name(), + embeddingModeId, + RefedTypeEnum.EMBEDDING.name(), + 1); + JSONObject config = new JSONObject(); + refInfo.setConfig(config.toJSONString()); + Long id = refManager.create(refInfo); + log.info( + "create project embedding ref info, refId: {}, refInfo: {}", + id, + JSON.toJSONString(refInfo)); + } + @Override public Project update(ProjectCreateRequest request) { Project project = projectRepository.queryById(request.getId()); - JSONObject config = setDatabase(request.getConfig(), project.getNamespace()); - setGraphStore(project, config, true); - config = setVectorDimensions(config, project); + if (StringUtils.isBlank(request.getTag())) { + request.setTag(project.getTag()); + } + if (!ProjectTagEnum.PUBLIC_NET.name().equalsIgnoreCase(request.getTag()) + && !StringUtils.equals(request.getTag(), project.getTag())) { + setLocalVectorizer(request); + } + String database = ""; + if (!ProjectTagEnum.PUBLIC_NET.name().equalsIgnoreCase(request.getTag())) { + JSONObject oldConfig = JSON.parseObject(project.getConfig()); + if (null != oldConfig && null != oldConfig.getJSONObject(SpgAppConstant.GRAPH_STORE)) { + database = + oldConfig + .getJSONObject(CommonConstants.GRAPH_STORE) + .getString(CommonConstants.DATABASE); + } + JSONObject config = request.getConfig(); + if (StringUtils.equals(project.getTag(), request.getTag())) { + JSONObject vectorConfig = oldConfig.getJSONObject(SpgAppConstant.VECTORIZER); + config.put(SpgAppConstant.VECTORIZER, vectorConfig); + } + // graphStore password special treatment + graphStoreDeserialization(config, oldConfig); + request.setConfig(config); + } + if (StringUtils.isBlank(request.getName())) { + request.setName(project.getName()); + } + JSONObject config = request.getConfig(); + if (!ProjectTagEnum.PUBLIC_NET.name().equalsIgnoreCase(request.getTag())) { + config = setDatabase(request.getConfig(), database); + setGraphStore(database, config, true); + config = setVectorDimensions(config, project); + config.remove(SpgAppConstant.MCP_SERVERS); + } else { + config.remove(SpgAppConstant.VECTORIZER); + config.remove(SpgAppConstant.GRAPH_STORE); + } Project update = new Project( request.getId(), @@ -84,23 +189,44 @@ public Project update(ProjectCreateRequest request) { request.getDescription(), null, null, - config.toJSONString()); + config.toJSONString(), + request.getTag()); + if (StringUtils.isNotBlank(request.getVisibility())) { + VisibilityEnum visibilityEnum = VisibilityEnum.getVisibilityEnum(request.getVisibility()); + update.setVisibility(visibilityEnum.name()); + } update = projectRepository.update(update); long start = System.currentTimeMillis(); - createSchema(request.getId()); log.info("createSchema cost {} ms", System.currentTimeMillis() - start); return update; } - private JSONObject setDatabase(String configStr, String namespace) { - JSONObject config = new JSONObject(); - if (StringUtils.isNotBlank(configStr)) { - config = JSONObject.parseObject(configStr); + private static void graphStoreDeserialization(JSONObject config, JSONObject oldConfig) { + JSONObject graphStore = config.getJSONObject(SpgAppConstant.GRAPH_STORE); + if (null == graphStore) { + return; + } + String password = graphStore.getString(SpgAppConstant.PASSWORD); + if (com.antgroup.openspg.common.util.StringUtils.equals( + password, SpgAppConstant.DEFAULT_VECTORIZER_API_KEY)) { + String oldPassword = + oldConfig.getJSONObject(SpgAppConstant.GRAPH_STORE).getString(SpgAppConstant.PASSWORD); + graphStore.put(SpgAppConstant.PASSWORD, oldPassword); + config.put(SpgAppConstant.GRAPH_STORE, graphStore); + } + } + + private JSONObject setDatabase(JSONObject config, String namespace) { + if (config == null) { + config = new JSONObject(); } if (config.containsKey(CommonConstants.GRAPH_STORE)) { - config - .getJSONObject(CommonConstants.GRAPH_STORE) - .put(CommonConstants.DATABASE, namespace.toLowerCase()); + JSONObject graphStore = config.getJSONObject(CommonConstants.GRAPH_STORE); + String uri = graphStore.getString("uri"); + if (StringUtils.isNotBlank(uri) && !uri.contains(CommonConstant.KGFABRIC_GRAPH_STORE)) { + namespace = namespace.toLowerCase(); + } + config.getJSONObject(CommonConstants.GRAPH_STORE).put(CommonConstants.DATABASE, namespace); } else { JSONObject graphStore = new JSONObject(); graphStore.put(CommonConstants.DATABASE, namespace.toLowerCase()); @@ -110,7 +236,7 @@ private JSONObject setDatabase(String configStr, String namespace) { } private JSONObject setVectorDimensions(JSONObject config, Project project) { - JSONObject oldConfig = JSONObject.parseObject(project.getConfig()); + JSONObject oldConfig = JSON.parseObject(project.getConfig()); String vectorDimensions = null; if (oldConfig.containsKey(CommonConstants.VECTORIZER)) { vectorDimensions = @@ -154,8 +280,9 @@ public Integer deleteById(Long projectId) { } public void deleteDatabase(Project project) { - JSONObject config = JSONObject.parseObject(project.getConfig()); - UriComponents uriComponents = UriComponentsBuilder.fromUriString(url).build(); + JSONObject config = JSON.parseObject(project.getConfig()); + UriComponents uriComponents = + UriComponentsBuilder.fromUriString(defaultValue.getGraphStoreUrl()).build(); String database = uriComponents.getQueryParams().getFirst(Neo4jConstants.DATABASE); String host = String.format( @@ -178,8 +305,9 @@ public void deleteDatabase(Project project) { driver.neo4jGraph.dropDatabase(dropDatabase); } - public void setGraphStore(Project project, JSONObject config, boolean createDatabase) { - UriComponents uriComponents = UriComponentsBuilder.fromUriString(url).build(); + public void setGraphStore(String namespace, JSONObject config, boolean createDatabase) { + UriComponents uriComponents = + UriComponentsBuilder.fromUriString(defaultValue.getGraphStoreUrl()).build(); String database = uriComponents.getQueryParams().getFirst(Neo4jConstants.DATABASE); String host = String.format( @@ -191,6 +319,7 @@ public void setGraphStore(Project project, JSONObject config, boolean createData JSONObject graphStore = config.getJSONObject(CommonConstants.GRAPH_STORE); if (graphStore.containsKey(Neo4jConstants.URI)) { host = graphStore.getString(Neo4jConstants.URI); + uriComponents = UriComponentsBuilder.fromUriString(host).build(); } else { graphStore.put(Neo4jConstants.URI, host); } @@ -204,9 +333,11 @@ public void setGraphStore(Project project, JSONObject config, boolean createData } else { graphStore.put(Neo4jConstants.PASSWORD, password); } - if (createDatabase) { + // TODO: 这里先跳过，kgfabric driver schema/索引同步打通后放开 @秉初 底层支持索引灵活变更的接口5月底给出，排期530开发 + if (!CommonConstant.KGFABRIC_GRAPH_STORE.equalsIgnoreCase(uriComponents.getScheme()) + && createDatabase) { Neo4jAdminUtils driver = new Neo4jAdminUtils(host, user, password, database); - String projectDatabase = project.getNamespace().toLowerCase(); + String projectDatabase = namespace.toLowerCase(); driver.neo4jGraph.createDatabase(projectDatabase); } } @@ -232,8 +363,13 @@ public List query(ProjectQueryRequest request) { } @Override - public Paged queryPaged(ProjectQueryRequest request, int start, int size) { - return projectRepository.queryPaged(request, start, size); + public List queryPageData(ProjectQueryRequest request, int start, int size) { + return projectRepository.queryPageData(request, start, size); + } + + @Override + public Long queryPageCount(ProjectQueryRequest request) { + return projectRepository.queryPageCount(request); } @Override @@ -251,4 +387,131 @@ public String getSearchEngineUrl(Long projectId) { public Project queryByNamespace(String namespace) { return projectRepository.queryByNamespace(namespace); } + + @Override + public void completionVectorizer(JSONObject projectConfig) { + JSONObject vectorizer = projectConfig.getJSONObject(CommonConstants.VECTORIZER); + if (vectorizer == null) { + return; + } + String modelId = vectorizer.getString(SpgAppConstant.MODEL_ID); + JSONObject llmInfo = userModelManager.getByModelId(modelId); + if (llmInfo != null) { + for (Map.Entry entry : llmInfo.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + vectorizer.put(key, value); + } + projectConfig.put(BuilderConstant.VECTORIZER, vectorizer); + } + } + + private JSONObject setDimensions(JSONObject config, String dimensions) { + if (config.containsKey(CommonConstants.VECTORIZER)) { + config + .getJSONObject(CommonConstants.VECTORIZER) + .put(CommonConstants.VECTOR_DIMENSIONS, dimensions); + } else { + JSONObject vectorizer = new JSONObject(); + vectorizer.put(CommonConstants.VECTOR_DIMENSIONS, dimensions); + config.put(CommonConstants.VECTORIZER, vectorizer); + } + return config; + } + + private void setLocalVectorizer(ProjectCreateRequest request) { + Boolean isKnext = request.getIsKnext(); + if (isKnext != null && isKnext) { + setLocalVectorizerKnext(request); + } else { + setLocalVectorizerPlatform(request); + } + } + + private void setLocalVectorizerPlatform(ProjectCreateRequest request) { + JSONObject jsonObject = request.getConfig(); + String modelId = getEmbeddingModelId(jsonObject); + AssertUtils.assertParamStringIsNotBlank("vectorizer", modelId); + JSONObject userModelConfig = userModelManager.getByModelId(modelId); + // Construct a new parameter object + JSONObject param = new JSONObject(); + JSONObject vector = new JSONObject(); + vector.putAll(jsonObject.getJSONObject(SpgAppConstant.VECTORIZER)); + param.put(SpgAppConstant.VECTORIZER, vector); + JSONObject vectorConfig = param.getJSONObject(CommonConstants.VECTORIZER); + vectorConfig.putAll(userModelConfig); + String dimensions = checkVectorizer(param); + // Set the dimensions to the original object + jsonObject = setDimensions(jsonObject, dimensions); + request.setConfig(jsonObject); + } + + private void setLocalVectorizerKnext(ProjectCreateRequest request) { + JSONObject jsonObject = request.getConfig(); + JSONObject vectorizer = jsonObject.getJSONObject(SpgAppConstant.VECTORIZER); + AssertUtils.assertParamObjectIsNotNull("tag is LOCAL, config.vectorizer", vectorizer); + String dimensions = checkVectorizer(jsonObject); + // Construct a new parameter object + UserModelDTO userModel = new UserModelDTO(); + userModel.setProvider("OpenAI"); + userModel.setName(request.getNamespace()); + userModel.setVisibility(VisibilityEnum.PRIVATE.name()); + userModel.setUserNo(request.getUserNo()); + JSONObject modelConfig = new JSONObject(); + modelConfig.putAll(vectorizer); + userModel.setConfig(modelConfig); + Map modelTypeMap = userModelManager.getModelTypeMap(); + AssertUtils.assertParamObjectIsNotNull("modelTypeMap", modelTypeMap); + JSONObject model = + userModelManager.getModelByProviderAndModel( + userModel.getProvider(), + userModel.getName(), + userModel.getVisibility(), + vectorizer.getString(SpgAppConstant.MODEL)); + if (model == null) { + Long num = userModelManager.insert(userModel, modelTypeMap); + if (num < 1) { + throw new IllegalParamsException("model insert failed"); + } + } + JSONObject modelInfo = + userModelManager.getModelByProviderAndModel( + userModel.getProvider(), + userModel.getName(), + userModel.getVisibility(), + vectorizer.getString(SpgAppConstant.MODEL)); + AssertUtils.assertParamObjectIsNotNull("modelInfo", modelInfo); + JSONObject newVectorizer = new JSONObject(); + newVectorizer.put(SpgAppConstant.MODEL_ID, modelInfo.getString(SpgAppConstant.MODEL_ID)); + // Set the dimensions to the original object + jsonObject.put(SpgAppConstant.VECTORIZER, newVectorizer); + jsonObject = setDimensions(jsonObject, dimensions); + request.setConfig(jsonObject); + } + + private String getEmbeddingModelId(JSONObject config) { + JSONObject vectorConfig = config.getJSONObject(SpgAppConstant.VECTORIZER); + if (vectorConfig != null) { + return vectorConfig.getString("modelId"); + } + return null; + } + + private String checkVectorizer(JSONObject config) { + if (config == null || !config.containsKey(CommonConstants.VECTORIZER)) { + return ""; + } + JSONObject vectorizerConfig = config.getJSONObject(CommonConstants.VECTORIZER); + PemjaConfig pemjaConfig = + new PemjaConfig( + defaultValue.getPythonExec(), + defaultValue.getPythonPaths(), + defaultValue.getPythonEnv(), + defaultValue.getSchemaUrlHost(), + null, + PythonInvokeMethod.BRIDGE_VECTORIZER_CHECKER, + Maps.newHashMap()); + Object result = PemjaUtils.invoke(pemjaConfig, JSON.toJSONString(vectorizerConfig)); + return result.toString(); + } } diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProviderParamManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProviderParamManagerImpl.java new file mode 100644 index 000000000..52f05ead8 --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ProviderParamManagerImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common.impl; + +import com.antgroup.openspg.server.biz.common.ModelDetailManager; +import com.antgroup.openspg.server.biz.common.ProviderParamManager; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import com.antgroup.openspg.server.common.model.providerparam.ProviderParam; +import com.antgroup.openspg.server.common.model.providerparam.ProviderParamQuery; +import com.antgroup.openspg.server.common.service.providerparam.ProviderParamRepository; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class ProviderParamManagerImpl implements ProviderParamManager { + + @Autowired private ProviderParamRepository providerParamRepository; + @Autowired private ModelDetailManager modelDetailManager; + + @Override + public Long insert(ProviderParam record) { + return providerParamRepository.insert(record); + } + + @Override + public int deleteById(Long id) { + return providerParamRepository.deleteById(id); + } + + @Override + public Long update(ProviderParam record) { + return providerParamRepository.update(record); + } + + @Override + public ProviderParam getById(Long id) { + return providerParamRepository.getById(id); + } + + @Override + public List query(String provider, String modelType) { + ProviderParamQuery record = new ProviderParamQuery(); + record.setProvider(provider); + record.setModelType(modelType); + return providerParamRepository.query(record); + } + + @Override + public ProviderParam getByProviderAndModelType(String provider, String modelType) { + ProviderParam providerParam = + providerParamRepository.getByProviderAndModelType(provider, modelType); + if (providerParam == null) { + return null; + } + ModelDetailQuery modelDetailQuery = new ModelDetailQuery(); + modelDetailQuery.setProvider(provider); + modelDetailQuery.setType(modelType); + List modelList = modelDetailManager.queryDTO(modelDetailQuery); + providerParam.setModel(modelList); + return providerParam; + } +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/RefManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/RefManagerImpl.java new file mode 100644 index 000000000..42ab29108 --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/RefManagerImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.biz.common.impl; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.util.DateTimeUtils; +import com.antgroup.openspg.server.biz.common.RefManager; +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.common.model.ref.RefTypeEnum; +import com.antgroup.openspg.server.common.model.ref.RefedTypeEnum; +import com.antgroup.openspg.server.common.service.ref.RefRepository; +import java.util.Date; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class RefManagerImpl implements RefManager { + + @Autowired private RefRepository refRepository; + + @Override + public Long create(RefInfo refInfo) { + // check type + RefTypeEnum.valueOf(refInfo.getRefType()); + RefedTypeEnum.valueOf(refInfo.getRefedType()); + return refRepository.insert(refInfo); + } + + @Override + public List getRefInfoByRef(String refId, RefTypeEnum refType) { + return refRepository.getRefInfoByRef(refId, refType); + } + + @Override + public List getRefInfoByRefed(String refedId, RefedTypeEnum refedType) { + return refRepository.getRefInfoByRefed(refedId, refedType); + } + + @Override + public List getRefInfoByRightMatchRefedId(String refedId) { + return refRepository.getRefInfoByRightMatchRefedId(refedId); + } + + @Override + public RefInfo getByUniqueKey( + String refId, RefTypeEnum refType, String refedId, RefedTypeEnum refedType) { + return refRepository.selectByUniqueKey(refId, refType.name(), refedId, refedType.name()); + } + + @Override + public int updateByPrimaryKeySelective(RefInfo refInfo) { + return refRepository.updateByPrimaryKeySelective(refInfo); + } + + @Override + public int updateByUniqueKey(RefInfo refInfo) { + return refRepository.updateByUniqueKey(refInfo); + } + + @Override + public int deleteById(Long id) { + return refRepository.deleteById(id); + } + + @Override + public int deleteByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return 0; + } + return refRepository.deleteByIds(ids); + } + + @Override + public int deleteByUniqueKey(RefInfo refInfo) { + return refRepository.deleteByUniqueKey(refInfo); + } + + @Override + public void recordApiKeyUsageInfo(String appId, String apiKey, String uri) { + try { + RefInfo refInfo = new RefInfo(); + refInfo.setRefId(appId); + refInfo.setRefType(RefTypeEnum.APP.name()); + refInfo.setRefedId(apiKey); + refInfo.setRefedType(RefedTypeEnum.API_KEY.name()); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("lastRequestUrl", uri); + jsonObject.put("lastRequestTime", DateTimeUtils.getDate2LongStr(new Date())); + refInfo.setConfig(jsonObject.toJSONString()); + refRepository.updateByUniqueKey(refInfo); + } catch (Exception e) { + log.error("recordApiKeyUsageInfo error, appId:{}, apiKey:{}, uri:{}", appId, apiKey, uri, e); + } + } +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/UserModelManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/UserModelManagerImpl.java new file mode 100644 index 000000000..bdd180fce --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/UserModelManagerImpl.java @@ -0,0 +1,535 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.common.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.antgroup.openspg.common.constants.SpgAppConstant; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.common.util.enums.PageModeEnum; +import com.antgroup.openspg.server.biz.common.ConfigManager; +import com.antgroup.openspg.server.biz.common.ModelDetailManager; +import com.antgroup.openspg.server.biz.common.ModelProviderManager; +import com.antgroup.openspg.server.biz.common.UserModelManager; +import com.antgroup.openspg.server.common.model.config.Config; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetail; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import com.antgroup.openspg.server.common.model.usermodel.UserModel; +import com.antgroup.openspg.server.common.model.usermodel.UserModelDTO; +import com.antgroup.openspg.server.common.model.usermodel.UserModelQuery; +import com.antgroup.openspg.server.common.service.usermodel.UserModelRepository; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +public class UserModelManagerImpl implements UserModelManager { + + private static final List sensitiveFields = + Arrays.asList("secret_id", "secret_key", "access_key", "api_key", "key"); + + @Autowired private UserModelRepository userModelRepository; + + @Autowired private ModelDetailManager modelDetailManager; + + @Autowired private ModelProviderManager modelProviderManager; + + @Autowired private ConfigManager configManager; + + @Override + public Long insert(UserModelDTO record, Map modelTypeMap) { + ModelProvider provider = modelProviderManager.getByProvider(record.getProvider()); + JSONObject config = record.getConfig(); + removeNullValue(config); + if (StringUtils.equals(provider.getPageMode(), PageModeEnum.SINGLE.name())) { + String modelType = config.getString(SpgAppConstant.MODEL_TYPE); + config.put( + SpgAppConstant.TYPE, + modelTypeMap.get(provider.getProvider() + StringUtils.UNDERLINE_SEPARATOR + modelType)); + } + + String uuid = UUID.randomUUID().toString().replace("-", ""); + + UserModel addUserModel = + new UserModel( + uuid, + record.getVisibility(), + record.getProvider(), + record.getName(), + record.getUserNo(), + record.getUserNo()); + + if (StringUtils.equals(provider.getPageMode(), PageModeEnum.ALL.name())) { + ModelDetailQuery modelDetailQuery = new ModelDetailQuery(); + modelDetailQuery.setProvider(provider.getProvider()); + List modelDetailList = modelDetailManager.query(modelDetailQuery); + + JSONArray jsonArray = new JSONArray(); + for (ModelDetail modelDetail : modelDetailList) { + JSONObject params = + modelDetail.getParams() != null ? modelDetail.getParams() : new JSONObject(); + if (!params.containsKey(SpgAppConstant.MODEL)) { + params.put(SpgAppConstant.MODEL, modelDetail.getName()); + } + if (!params.containsKey(SpgAppConstant.MODEL_TYPE)) { + params.put(SpgAppConstant.MODEL_TYPE, modelDetail.getType()); + } + JSONObject completeConfig = new JSONObject(); + completeConfig.put( + SpgAppConstant.TYPE, + modelTypeMap.get( + provider.getProvider() + StringUtils.UNDERLINE_SEPARATOR + modelDetail.getType())); + completeConfig.putAll(config); + completeConfig.putAll(params); + generateModelId(uuid, completeConfig); + jsonArray.add(completeConfig); + } + addUserModel.setConfig( + JSON.toJSONString(jsonArray, SerializerFeature.DisableCircularReferenceDetect)); + return userModelRepository.insert(addUserModel); + } else if (StringUtils.equals(provider.getPageMode(), PageModeEnum.SINGLE.name())) { + List userModels = getByProviderAndName(provider.getProvider(), record.getName()); + UserModel userModel = + userModels.stream() + .filter(m -> StringUtils.equals(m.getVisibility(), addUserModel.getVisibility())) + .findFirst() + .orElse(null); + if (userModel != null) { + JSONArray modelList = userModel.getModelList(); + generateModelId(userModel.getInstanceId(), config); + modelList.add(config); + userModel.setConfig( + JSON.toJSONString(modelList, SerializerFeature.DisableCircularReferenceDetect)); + userModelRepository.update(userModel); + return userModel.getId(); + } else { + JSONArray jsonArray = new JSONArray(); + generateModelId(uuid, config); + jsonArray.add(config); + addUserModel.setConfig( + JSON.toJSONString(jsonArray, SerializerFeature.DisableCircularReferenceDetect)); + return userModelRepository.insert(addUserModel); + } + } + return 0L; + } + + private void removeNullValue(JSONObject config) { + List removeKey = new ArrayList<>(); + for (String key : config.keySet()) { + Object value = config.get(key); + if (value instanceof Integer) { + if (value == null) { + removeKey.add(key); + } + } else if (value instanceof String && StringUtils.isBlank((String) value)) { + removeKey.add(key); + } + } + removeKey.forEach(key -> config.remove(key)); + } + + @Override + public int deleteById(Long id) { + return userModelRepository.deleteById(id); + } + + @Override + public int deleteByIds(List ids) { + return userModelRepository.deleteByIds(ids); + } + + @Override + public Long update(UserModel record) { + return userModelRepository.update(record); + } + + @Override + public UserModel getById(Long id) { + return userModelRepository.getById(id); + } + + @Override + public List query(UserModelQuery record) { + return userModelRepository.query(record); + } + + @Override + public List getByProviderAndName(String provider, String name) { + return userModelRepository.getByProviderAndName(provider, name); + } + + public void generateModelId(String uuid, JSONObject config) { + if (config == null) { + return; + } + if (!config.containsKey(SpgAppConstant.MODEL_ID) + || !StringUtils.equals(uuid, config.getString(SpgAppConstant.MODEL_ID))) { + String model = config.getString(SpgAppConstant.MODEL); + config.put(SpgAppConstant.MODEL_ID, uuid + "@" + model); + } + } + + @Override + public Long updateApiKey(UserModelDTO request) { + UserModel userModel = getById(request.getId()); + if (userModel == null) { + return 0L; + } + userModel.setModifier(request.getUserNo()); + JSONObject configJson = request.getConfig(); + JSONArray modelList = userModel.getModelList(); + for (int i = 0; i < modelList.size(); i++) { + JSONObject modelJson = modelList.getJSONObject(i); + modelJson.putAll(configJson); + userModel.setConfig(JSON.toJSONString(modelList)); + } + return update(userModel); + } + + @Override + public List> list( + String modelType, String queryStr, String modelId, String userNo) { + List userModelList = userModelRepository.selectUserPrivateOrPublic(userNo); + Map modelMap = + userModelList.stream() + .collect(Collectors.toMap(UserModel::getInstanceId, userModel -> userModel)); + if (StringUtils.isNotBlank(modelId) && modelId.split("@").length == 2) { + String[] split = modelId.split("@"); + UserModel addModel = getByInstanceId(split[0]); + if (addModel != null) { + JSONArray modelList = addModel.getModelList(); + for (int i = 0; i < modelList.size(); i++) { + JSONObject jsonObject = modelList.getJSONObject(i); + if (StringUtils.equals(modelId, jsonObject.getString(SpgAppConstant.MODEL_ID))) { + UserModel userModel = modelMap.get(addModel.getInstanceId()); + if (userModel != null + && StringUtils.equals(userModel.getInstanceId(), addModel.getInstanceId())) { + break; + } + JSONArray modelModelList = + userModel != null ? userModel.getModelList() : new JSONArray(); + modelModelList.add(jsonObject); + addModel.setModelList(modelModelList); + modelMap.put(addModel.getInstanceId(), addModel); + break; + } + } + } + } + List userModels = new ArrayList<>(modelMap.values()); + if (CollectionUtils.isEmpty(userModels)) { + return Collections.emptyList(); + } + List providerList = + userModels.stream().map(UserModel::getProvider).collect(Collectors.toList()); + List providers = modelProviderManager.selectByProviders(providerList); + if (CollectionUtils.isEmpty(providers)) { + return Collections.emptyList(); + } + Map providerMap = + providers.stream() + .collect( + Collectors.toMap( + ModelProvider::getProvider, + provider -> ModelProvider.modelProvider(provider), + (v1, v2) -> v1)); + + Map> map = new HashMap<>(); + for (UserModel userModel : userModels) { + String onlyName = + userModel.getProvider() + StringUtils.UNDERLINE_SEPARATOR + userModel.getName(); + ModelProvider provider = providerMap.get(userModel.getProvider()); + JSONArray modelList = userModel.getModelList(); + + JSONArray resultModelList = new JSONArray(); + if (map.containsKey(onlyName)) { + Map providerInfo = map.get(onlyName); + if (providerInfo.containsKey(SpgAppConstant.MODEL)) { + resultModelList = (JSONArray) providerInfo.get(SpgAppConstant.MODEL); + } + } + + if (StringUtils.isNotBlank(modelType)) { + modelList = + modelList.stream() + .filter( + model -> + StringUtils.equals( + modelType, ((JSONObject) model).getString(SpgAppConstant.MODEL_TYPE))) + .collect(Collectors.toCollection(JSONArray::new)); + } + if (StringUtils.isNotBlank(queryStr)) { + modelList = + modelList.stream() + .filter( + model -> + StringUtils.contains(userModel.getName(), queryStr) + || StringUtils.contains( + ((JSONObject) model).getString(SpgAppConstant.MODEL), queryStr)) + .collect(Collectors.toCollection(JSONArray::new)); + } + for (int i = 0; i < modelList.size(); i++) { + JSONObject modelJson = modelList.getJSONObject(i); + desensitizedFieldSensitive(modelJson); + modelJson.put(SpgAppConstant.VISIBILITY, userModel.getVisibility()); + resultModelList.add(modelJson); + } + if (CollectionUtils.isEmpty(resultModelList)) { + continue; + } + Map providerModel = new HashMap<>(); + providerModel.put("model", resultModelList); + providerModel.putAll(JSON.parseObject(JSON.toJSONString(provider))); + providerModel.put("id", userModel.getId()); + providerModel.put("visibility", userModel.getVisibility()); + providerModel.put("userNo", userModel.getUserNo()); + providerModel.put("name", userModel.getName()); + if (StringUtils.equals(provider.getPageMode(), PageModeEnum.ALL.name())) { + JSONObject jsonObject = resultModelList.getJSONObject(0); + providerModel.put("config", jsonObject); + } + map.put(onlyName, providerModel); + } + List> list = map.values().stream().collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(list)) { + list.sort( + (mapA, mapB) -> { + Long idA = (Long) mapA.get("id"); + Long idB = (Long) mapB.get("id"); + return idB.compareTo(idA); // 降序排序 + }); + } + return list; + } + + @Override + public JSONObject getByModelId(String modelId) { + return userModelRepository.getByModelId(modelId); + } + + @Override + public Boolean deleteModel(Long id, String modelId) { + UserModel userModel = getById(id); + JSONArray modelList = userModel.getModelList(); + for (int i = 0; i < modelList.size(); i++) { + JSONObject modelJson = modelList.getJSONObject(i); + if (StringUtils.equals(modelId, modelJson.getString(SpgAppConstant.MODEL_ID))) { + modelList.remove(i); + if (modelList.size() <= 0) { + return deleteById(id) > 0; + } else { + userModel.setConfig(JSON.toJSONString(modelList)); + return update(userModel) > 0; + } + } + } + return false; + } + + @Transactional(value = "transactionManager", rollbackFor = Exception.class) + @Override + public Boolean updateModelVisibility(String modelId, String visibility, JSONObject customize) { + String[] split = modelId.split("@"); + if (split.length < 2) { + return false; + } + String instanceId = split[0]; + String configModel = split[1]; + UserModel userModel = userModelRepository.getByInstanceId(instanceId); + if (userModel == null) { + return false; + } + if (StringUtils.equals(userModel.getVisibility(), visibility)) { + JSONArray modelList = userModel.getModelList(); + for (int i = 0; i < modelList.size(); i++) { + JSONObject modelConfig = modelList.getJSONObject(i); + modelConfig.remove(SpgAppConstant.CUSTOMIZE); + if (customize != null) { + modelConfig.put(SpgAppConstant.CUSTOMIZE, customize); + } + } + userModel.setConfig( + JSON.toJSONString(modelList, SerializerFeature.DisableCircularReferenceDetect)); + return update(userModel) > 0; + } + JSONObject modelInfo = null; + JSONArray modelList = userModel.getModelList(); + for (int i = 0; i < modelList.size(); i++) { + JSONObject jsonObject = modelList.getJSONObject(i); + if (jsonObject.containsKey(SpgAppConstant.MODEL) + && StringUtils.equals(configModel, jsonObject.getString(SpgAppConstant.MODEL))) { + modelInfo = jsonObject; + } + } + if (modelInfo == null) { + return false; + } + modelList.remove(modelInfo); + modelInfo.remove(SpgAppConstant.CUSTOMIZE); + if (customize != null) { + modelInfo.put(SpgAppConstant.CUSTOMIZE, customize); + } + List oldUserModelList = + getByProviderAndName(userModel.getProvider(), userModel.getName()); + Optional first = + oldUserModelList.stream() + .filter(model -> StringUtils.equals(visibility, model.getVisibility())) + .findFirst(); + if (first.isPresent()) { + UserModel model = first.get(); + JSONArray addModelList = model.getModelList(); + if (CollectionUtils.isNotEmpty(addModelList) + && addModelList.toJavaList(Map.class).stream() + .noneMatch( + map -> StringUtils.equals((String) map.get(SpgAppConstant.MODEL), configModel))) { + modelInfo.put( + SpgAppConstant.MODEL_ID, + model.getInstanceId() + "@" + modelInfo.getString(SpgAppConstant.MODEL)); + addModelList.add(modelInfo); + UserModel updateModel = new UserModel(); + updateModel.setId(model.getId()); + updateModel.setConfig(JSON.toJSONString(addModelList)); + userModelRepository.update(updateModel); + } + } else { + String uuid = UUID.randomUUID().toString().replace("-", ""); + UserModel addUserModel = + new UserModel( + uuid, + visibility, + userModel.getProvider(), + userModel.getName(), + userModel.getUserNo(), + userModel.getUserNo()); + modelInfo.put( + SpgAppConstant.MODEL_ID, uuid + "@" + modelInfo.getString(SpgAppConstant.MODEL)); + addUserModel.setConfig(JSON.toJSONString(Arrays.asList(modelInfo))); + userModelRepository.insert(addUserModel); + } + if (CollectionUtils.isNotEmpty(modelList)) { + UserModel updateModel = new UserModel(); + updateModel.setId(userModel.getId()); + updateModel.setConfig(JSON.toJSONString(modelList)); + updateModel.setModifier(userModel.getUserNo()); + update(updateModel); + } else { + userModelRepository.deleteById(userModel.getId()); + } + return true; + } + + @Override + public UserModel getByInstanceId(String instanceId) { + return userModelRepository.getByInstanceId(instanceId); + } + + @Override + public Long updateBaseInfoByIds( + List ids, String name, String visibility, String userNo, String config) { + return userModelRepository.updateBaseInfoByIds(ids, name, visibility, userNo, config); + } + + @Override + public List selectUserPrivateOrPublic(String userNo) { + return userModelRepository.selectUserPrivateOrPublic(userNo); + } + + @Override + public JSONObject desensitizedFieldSensitive(JSONObject config) { + if (config == null) { + return null; + } + for (String key : config.keySet()) { + if (sensitiveFields.contains(key)) { + config.put(key, SpgAppConstant.DEFAULT_VECTORIZER_API_KEY); + } + } + return config; + } + + @Override + public JSONObject sensitiveFieldRecovery(JSONObject config, JSONObject oldConfig) { + if (config == null || oldConfig == null) { + return null; + } + List recoveryFields = Lists.newArrayList(); + for (Map.Entry entry : config.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue().toString(); + if (StringUtils.equals(value, SpgAppConstant.DEFAULT_VECTORIZER_API_KEY)) { + recoveryFields.add(key); + } + } + if (CollectionUtils.isEmpty(recoveryFields)) { + return config; + } + for (Map.Entry entry : oldConfig.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue().toString(); + if (recoveryFields.contains(key)) { + config.put(key, value); + } + } + return config; + } + + @Override + public Map getModelTypeMap() { + Config providerBaseInfo = configManager.query("PROVIDER_BASE_INFO", "1"); + if (providerBaseInfo == null) { + return null; + } + JSONObject providerInfo = (JSONObject) providerBaseInfo.getConfig(); + return providerInfo.getJSONObject(SpgAppConstant.MODEL_TYPE); + } + + @Override + public JSONObject getModelByProviderAndModel( + String provider, String name, String visibility, String model) { + List userModelList = getByProviderAndName(provider, name); + if (CollectionUtils.isEmpty(userModelList)) { + return null; + } + Optional first = + userModelList.stream() + .filter(userModel -> StringUtils.equals(userModel.getVisibility(), visibility)) + .findFirst(); + if (!first.isPresent()) { + return null; + } + UserModel userModel = first.get(); + JSONArray modelList = userModel.getModelList(); + for (int i = 0; i < modelList.size(); i++) { + JSONObject jsonObject = modelList.getJSONObject(i); + if (StringUtils.equals(model, jsonObject.getString(SpgAppConstant.MODEL))) { + return jsonObject; + } + } + return null; + } +} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/AssertUtils.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/AssertUtils.java index b867a84b9..fd28c8e31 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/AssertUtils.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/AssertUtils.java @@ -15,6 +15,7 @@ import com.antgroup.openspg.server.common.model.exception.IllegalParamsException; import java.util.Collection; +import java.util.regex.Pattern; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -146,4 +147,16 @@ public static void assertIntegerIsBetween(String objectName, Integer object, int throw new IllegalParamsException(objectName + " must <= " + max + " and >=" + min); } } + /** + * Asserts that a string matches a given regular expression. + * + * @param value The string to be validated + * @param regex The regular expression to match + * @param message The error message to throw if the validation fails + */ + public static void assertValidPattern(String value, String regex, String message) { + if (value == null || !Pattern.matches(regex, value)) { + throw new IllegalParamsException(message); + } + } } diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/RegexPattern.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/RegexPattern.java new file mode 100644 index 000000000..c91aad72d --- /dev/null +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/util/RegexPattern.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +/* + * Ant Group + * Copyright (c) 2004-2025 All Rights Reserved. + */ +package com.antgroup.openspg.server.biz.common.util; + +public class RegexPattern { + + public static final String ALIAS_PATTERN = "^[a-zA-Z0-9]{1,32}$"; +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/ConceptInstanceManager.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/ConceptInstanceManager.java index 4dd62e473..2c97676db 100644 --- a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/ConceptInstanceManager.java +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/ConceptInstanceManager.java @@ -23,5 +23,6 @@ public interface ConceptInstanceManager { ConceptLevelInstanceResponse queryConceptLevelInstance(ConceptLevelInstanceRequest request); - List query(String conceptType, Set conceptInstanceIds); + List query( + Long projectId, String conceptType, Set conceptInstanceIds); } diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/impl/ConceptInstanceManagerImpl.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/impl/ConceptInstanceManagerImpl.java index 64298d1d0..a591461b9 100644 --- a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/impl/ConceptInstanceManagerImpl.java +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/impl/ConceptInstanceManagerImpl.java @@ -16,51 +16,58 @@ import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClientDriverManager; import com.antgroup.openspg.cloudext.interfaces.graphstore.LPGDataQueryService; import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.BatchVertexLPGRecordQuery; -import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.ObjectVertexRecordQuery; +import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.OneHopLPGRecordQuery; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.Direction; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.struct.GraphLPGRecordStruct; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName; +import com.antgroup.openspg.core.schema.model.identifier.ConceptIdentifier; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.type.BaseSPGType; import com.antgroup.openspg.core.schema.model.type.ConceptType; import com.antgroup.openspg.server.api.facade.dto.schema.request.ConceptLevelInstanceRequest; import com.antgroup.openspg.server.api.facade.dto.schema.response.ConceptInstanceResponse; import com.antgroup.openspg.server.api.facade.dto.schema.response.ConceptLevelInstanceResponse; +import com.antgroup.openspg.server.biz.common.ProjectManager; import com.antgroup.openspg.server.biz.schema.ConceptInstanceManager; -import com.antgroup.openspg.server.common.service.config.AppEnvConfig; +import com.antgroup.openspg.server.common.model.project.Project; import com.antgroup.openspg.server.core.schema.service.type.SPGTypeService; import com.google.common.collect.Sets; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +@Slf4j @Service public class ConceptInstanceManagerImpl implements ConceptInstanceManager { - @Autowired private AppEnvConfig appEnvConfig; - @Autowired private SPGTypeService spgTypeService; + @Autowired private ProjectManager projectManager; + @Override public ConceptLevelInstanceResponse queryConceptLevelInstance( ConceptLevelInstanceRequest request) { - LPGDataQueryService graphStoreClient = getGraphStoreClient(); - + Long projectId = request.getProjectId(); + if (projectId == null) { + projectId = getProjectId(request.getConceptType()); + } + String graphStoreUrl = projectManager.getGraphStoreUrl(projectId); + LPGDataQueryService graphStoreClient = + (LPGDataQueryService) GraphStoreClientDriverManager.getClient(graphStoreUrl); BaseSPGType spgType = spgTypeService.querySPGTypeByIdentifier(SPGTypeIdentifier.parse(request.getConceptType())); if (spgType == null || !spgType.isConceptType()) { throw new IllegalArgumentException( String.format("%s is not a concept type", request.getConceptType())); } - ConceptType conceptType = (ConceptType) spgType; - GraphLPGRecordStruct graph = (GraphLPGRecordStruct) graphStoreClient.queryRecord( - new ObjectVertexRecordQuery( + new OneHopLPGRecordQuery( request.getRootConceptInstance(), request.getConceptType(), Sets.newHashSet( @@ -73,9 +80,14 @@ public ConceptLevelInstanceResponse queryConceptLevelInstance( } @Override - public List query(String conceptType, Set conceptInstanceIds) { - LPGDataQueryService graphStoreClient = getGraphStoreClient(); - + public List query( + Long projectId, String conceptType, Set conceptInstanceIds) { + if (projectId == null) { + projectId = getProjectId(conceptType); + } + String graphStoreUrl = projectManager.getGraphStoreUrl(projectId); + LPGDataQueryService graphStoreClient = + (LPGDataQueryService) GraphStoreClientDriverManager.getClient(graphStoreUrl); GraphLPGRecordStruct graph = (GraphLPGRecordStruct) graphStoreClient.queryRecord( @@ -83,9 +95,14 @@ public List query(String conceptType, Set conce return toResponse(graph); } - private LPGDataQueryService getGraphStoreClient() { - return (LPGDataQueryService) - GraphStoreClientDriverManager.getClient(appEnvConfig.getGraphStoreUrl()); + private Long getProjectId(String conceptType) { + Project project = + projectManager.queryByNamespace(SPGTypeIdentifier.parse(conceptType).getNamespace()); + if (project == null) { + throw new IllegalArgumentException( + String.format("can not find project by namespace %s", conceptType)); + } + return project.getId(); } private ConceptLevelInstanceResponse toResponse( @@ -106,6 +123,7 @@ private List toResponse(GraphLPGRecordStruct graph) { instance.setProperties(vertex.toPropertyMap()); return instance; }) + .filter(it -> !ConceptIdentifier.ROOT.equals(it.getId())) .collect(Collectors.toList()); } } diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/EdgeTypeModel.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/EdgeTypeModel.java new file mode 100644 index 000000000..733ef898e --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/EdgeTypeModel.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import com.antgroup.openspg.core.schema.model.OntologyId; +import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; +import java.util.List; +import lombok.Data; + +@Data +public class EdgeTypeModel { + private OntologyId ontologyId; + private String name; + private String nameZh; + private String desc; + private Boolean inherited; + private Boolean semanticRelation; + private String sourceType; + private String targetType; + private LogicalRule rule; + private List properties; +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/NodeTypeModel.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/NodeTypeModel.java new file mode 100644 index 000000000..6f16ba0e5 --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/NodeTypeModel.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import com.antgroup.openspg.core.schema.model.OntologyId; +import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; +import java.util.List; +import lombok.Data; + +@Data +public class NodeTypeModel { + + private OntologyId ontologyId; + + private String name; + + private String nameZh; + + private String desc; + /** @see SPGTypeEnum */ + private String type; + + private String parentName; + + private String hypernymPredicate; + + private List properties; + + private List relations; +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/PropertyModel.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/PropertyModel.java new file mode 100644 index 000000000..53a34470b --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/PropertyModel.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import com.antgroup.openspg.core.schema.model.OntologyId; +import com.antgroup.openspg.core.schema.model.constraint.Constraint; +import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; +import java.util.List; +import lombok.Data; + +@Data +public class PropertyModel { + private OntologyId ontologyId; + private String name; + private String nameZh; + private String type; + private String desc; + private Boolean inherited; + private String index; + private LogicalRule rule; + private Constraint constraint; + private List subProperties; +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaCompareUtil.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaCompareUtil.java new file mode 100644 index 000000000..5f4ee423c --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaCompareUtil.java @@ -0,0 +1,755 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import static com.antgroup.openspg.reasoner.common.constants.Constants.OPEN_CONCEPT_HYPERNYM_PREDICATE; + +import com.antgroup.openspg.common.util.enums.AdvancedTypeEnum; +import com.antgroup.openspg.common.util.enums.BasicTypeEnum; +import com.antgroup.openspg.common.util.exception.SpgException; +import com.antgroup.openspg.common.util.exception.message.SpgMessageEnum; +import com.antgroup.openspg.core.schema.model.BasicInfo; +import com.antgroup.openspg.core.schema.model.SchemaExtInfo; +import com.antgroup.openspg.core.schema.model.alter.AlterOperationEnum; +import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier; +import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; +import com.antgroup.openspg.core.schema.model.predicate.IndexTypeEnum; +import com.antgroup.openspg.core.schema.model.predicate.Property; +import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig; +import com.antgroup.openspg.core.schema.model.predicate.PropertyGroupEnum; +import com.antgroup.openspg.core.schema.model.predicate.Relation; +import com.antgroup.openspg.core.schema.model.predicate.SubProperty; +import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; +import com.antgroup.openspg.core.schema.model.semantic.SystemPredicateEnum; +import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; +import com.antgroup.openspg.core.schema.model.type.ConceptLayerConfig; +import com.antgroup.openspg.core.schema.model.type.ConceptType; +import com.antgroup.openspg.core.schema.model.type.EntityType; +import com.antgroup.openspg.core.schema.model.type.EventType; +import com.antgroup.openspg.core.schema.model.type.IndexType; +import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; +import com.antgroup.openspg.core.schema.model.type.SPGTypeAdvancedConfig; +import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; +import com.antgroup.openspg.core.schema.model.type.SPGTypeRef; +import com.antgroup.openspg.server.core.schema.service.type.model.BuiltInPropertyEnum; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.StringUtils; + +/** schema compare util */ +@Slf4j +public class SchemaCompareUtil { + + /** + * compare schema change + * + * @param namespace + * @param oldTypes + * @param newTypes + * @return SchemaChangeDTO + */ + public SchemaChangeDTO compare( + String namespace, List oldTypes, List newTypes) { + List addAdvancedType = new ArrayList<>(); + List updateAdvancedType = new ArrayList<>(); + List deleteAdvancedType = new ArrayList<>(); + SchemaChangeDTO result = + new SchemaChangeDTO(addAdvancedType, updateAdvancedType, deleteAdvancedType); + List oldTypeNames = Lists.newArrayList(); + List newTypeNames = Lists.newArrayList(); + Map oldMap = Maps.newHashMap(); + Map newMap = Maps.newHashMap(); + Map name2Type = Maps.newHashMap(); + if (CollectionUtils.isEmpty(newTypes)) { + convertToAddOrDelete( + namespace, oldTypes, deleteAdvancedType, name2Type, AlterOperationEnum.DELETE); + return result; + } + newTypes.forEach( + n -> { + newTypeNames.add(n.getName()); + newMap.put(n.getName(), n); + name2Type.put(n.getName(), n.getType()); + }); + + List intersection = new ArrayList<>(newTypeNames); + List newCreate = new ArrayList<>(newTypeNames); + List delete = new ArrayList<>(); + List create = new ArrayList<>(); + if (CollectionUtils.isEmpty(oldTypes)) { + convertToAddOrDelete( + namespace, newTypes, addAdvancedType, name2Type, AlterOperationEnum.CREATE); + return result; + } + oldTypes.forEach( + o -> { + oldTypeNames.add(o.getName()); + oldMap.put(o.getName(), o); + name2Type.put(o.getName(), o.getType()); + // 获取新增和删除的类型 + newCreate.remove(o.getName()); + if (!newTypeNames.contains(o.getName())) { + delete.add(o); + } + }); + intersection.retainAll(oldTypeNames); + for (String s : newCreate) { + if (!oldTypeNames.contains(s)) { + create.add(newMap.get(s)); + } + } + for (String s : intersection) { + NodeTypeModel o = oldMap.get(s); + NodeTypeModel n = newMap.get(s); + convertToUpdate(namespace, o, n, name2Type, updateAdvancedType); + } + convertToAddOrDelete( + namespace, delete, deleteAdvancedType, name2Type, AlterOperationEnum.DELETE); + convertToAddOrDelete(namespace, create, addAdvancedType, name2Type, AlterOperationEnum.CREATE); + return result; + } + + /** + * update property + * + * @param namespace + * @param o + * @param n + * @param updateAdvancedType + */ + private static void convertToUpdate( + String namespace, + NodeTypeModel o, + NodeTypeModel n, + Map name2Type, + List updateAdvancedType) { + AdvancedTypeEnum nType = AdvancedTypeEnum.valueOf(n.getType()); + AdvancedTypeEnum oType = AdvancedTypeEnum.valueOf(o.getType()); + if (!AdvancedTypeEnum.canConvert(oType, nType)) { + log.warn("update node type, nodeName={}, oldType={}", n.getName(), o.getType()); + throw new SpgException(SpgMessageEnum.SCHEMA_CHANGE_NODE_TYPE); + } + SPGTypeIdentifier spgTypeIdentifier = new SPGTypeIdentifier(namespace, n.getName()); + BasicInfo basicInfo = new BasicInfo<>(spgTypeIdentifier, n.getNameZh(), n.getDesc()); + ParentTypeInfo parentTypeInfo = ParentTypeInfo.THING; + if (StringUtils.isNotBlank(n.getParentName())) { + parentTypeInfo = + new ParentTypeInfo(null, null, SPGTypeIdentifier.parse(n.getParentName()), null); + } + SPGTypeAdvancedConfig spgTypeAdvancedConfig = new SPGTypeAdvancedConfig(); + List props = n.getProperties(); + AdvancedTypeEnum advancedTypeEnum = AdvancedTypeEnum.valueOf(n.getType()); + BaseAdvancedType type; + switch (advancedTypeEnum) { + case ENTITY_TYPE: + type = new EntityType(basicInfo, parentTypeInfo, null, null, spgTypeAdvancedConfig); + break; + case EVENT_TYPE: + type = new EventType(basicInfo, parentTypeInfo, null, null, spgTypeAdvancedConfig); + break; + case INDEX_TYPE: + type = new IndexType(basicInfo, parentTypeInfo, null, null, spgTypeAdvancedConfig); + break; + case CONCEPT_TYPE: + if (!o.getHypernymPredicate().equalsIgnoreCase(n.getHypernymPredicate())) { + log.warn( + "update node type, nodeName={}, oldHypernymPredicate={}, newHypernymPredicate={}", + n.getName(), + o.getHypernymPredicate(), + n.getHypernymPredicate()); + throw new SpgException(SpgMessageEnum.SCHEMA_CHANGE_HYPERNYM_PREDICATE); + } + type = + new ConceptType( + basicInfo, + parentTypeInfo, + null, + null, + spgTypeAdvancedConfig, + new ConceptLayerConfig(n.getHypernymPredicate(), null), + null, + null); + break; + default: + throw new RuntimeException("unsupported advanced type: " + n.getType()); + } + type.setExtInfo(new SchemaExtInfo()); + updateProperty(namespace, o, props, name2Type, type); + updateRelation(namespace, o, n.getRelations(), name2Type, type); + type.setOntologyId(o.getOntologyId()); + type.setAlterOperation(AlterOperationEnum.UPDATE); + updateAdvancedType.add(type); + } + + /** + * update Relation + * + * @param namespace + * @param o old node type + * @param relations new node type relations + * @param advancedType final advanced type + */ + private static void updateRelation( + String namespace, + NodeTypeModel o, + List relations, + Map name2Type, + BaseAdvancedType advancedType) { + if (CollectionUtils.isEmpty(relations) && CollectionUtils.isEmpty(o.getRelations())) { + return; + } + List relationList = Lists.newArrayList(); + advancedType.setRelations(relationList); + List oldRelations = o.getRelations(); + Map oldRelationMap = Maps.newHashMap(); + List oldRelationSPO = Lists.newArrayList(); + if (CollectionUtils.isNotEmpty(oldRelations)) { + for (EdgeTypeModel oldRelation : oldRelations) { + String spo = + String.format( + "%s_%s_%s", o.getName(), oldRelation.getName(), oldRelation.getTargetType()); + oldRelationMap.put(spo, oldRelation); + oldRelationSPO.add(spo); + } + } + List delete = new ArrayList<>(oldRelationSPO); + if (CollectionUtils.isNotEmpty(relations) && !(advancedType instanceof ConceptType)) { + relationCreateOrUpdate( + namespace, o, relations, advancedType, oldRelationMap, delete, relationList, name2Type); + } + if (CollectionUtils.isEmpty(delete)) { + return; + } + delete.forEach( + spo -> { + EdgeTypeModel typeModel = oldRelationMap.get(spo); + SPGTypeRef targetType = getTargetType(typeModel.getTargetType(), namespace, name2Type); + Relation relation = + new Relation( + new BasicInfo<>( + new PredicateIdentifier(typeModel.getName()), + typeModel.getNameZh(), + typeModel.getDesc()), + advancedType.toRef(), + targetType, + false, + new PropertyAdvancedConfig(), + false); + relation.setOntologyId(typeModel.getOntologyId()); + boolean semanticRelation = + typeModel.getSemanticRelation() == null ? false : typeModel.getSemanticRelation(); + boolean isConceptRelation = + advancedType instanceof ConceptType + && OPEN_CONCEPT_HYPERNYM_PREDICATE.contains(typeModel.getName()); + if (isConceptRelation || semanticRelation) { + relation.setAlterOperation(null); + } else { + relation.setAlterOperation(AlterOperationEnum.DELETE); + } + relationList.add(relation); + }); + } + + /** + * relation create or update + * + * @param namespace + * @param o + * @param relations + * @param advancedType + * @param oldRelationMap + * @param delete + * @param relationList + */ + private static void relationCreateOrUpdate( + String namespace, + NodeTypeModel o, + List relations, + BaseAdvancedType advancedType, + Map oldRelationMap, + List delete, + List relationList, + Map name2Type) { + for (EdgeTypeModel edgeTypeModel : relations) { + String spo = + String.format( + "%s_%s_%s", o.getName(), edgeTypeModel.getName(), edgeTypeModel.getTargetType()); + SPGTypeRef spgTypeRef = getTargetType(edgeTypeModel.getTargetType(), namespace, name2Type); + Relation relation = + new Relation( + new BasicInfo<>( + new PredicateIdentifier(edgeTypeModel.getName()), + edgeTypeModel.getNameZh(), + edgeTypeModel.getDesc()), + advancedType.toRef(), + spgTypeRef, + false, + new PropertyAdvancedConfig(), + false); + relation.setExtInfo(new SchemaExtInfo()); + EdgeTypeModel typeModel = oldRelationMap.get(spo); + if ((typeModel != null && CollectionUtils.isNotEmpty(typeModel.getProperties())) + || CollectionUtils.isNotEmpty(edgeTypeModel.getProperties())) { + updateRelationProperty(typeModel, edgeTypeModel.getProperties(), relation); + } + if (typeModel == null) { + relation.getAdvancedConfig().setLogicalRule(setLogicalRule(null, edgeTypeModel.getRule())); + relation.setAlterOperation(AlterOperationEnum.CREATE); + } else { + delete.remove(spo); + relation + .getAdvancedConfig() + .setLogicalRule(setLogicalRule(typeModel.getRule(), edgeTypeModel.getRule())); + relation.setOntologyId(typeModel.getOntologyId()); + relation.setAlterOperation(AlterOperationEnum.UPDATE); + } + relationList.add(relation); + } + } + + /** + * update property + * + * @param namespace + * @param o old node type + * @param props new properties + * @param advancedType + */ + private static void updateProperty( + String namespace, + NodeTypeModel o, + List props, + Map name2Type, + BaseAdvancedType advancedType) { + List propertyList = Lists.newArrayList(); + Map oldPropMap = Maps.newHashMap(); + List oldPro = Lists.newArrayList(); + List oldProps = o.getProperties(); + for (PropertyModel oldProp : oldProps) { + oldPropMap.put(oldProp.getName(), oldProp); + oldPro.add(oldProp.getName()); + } + props = CollectionUtils.isEmpty(props) ? Lists.newArrayList() : props; + List deleteProps = new ArrayList<>(oldPro); + for (PropertyModel newProp : props) { + PropertyModel oldProp = oldPropMap.get(newProp.getName()); + if (oldProp != null && !oldProp.getType().equals(newProp.getType())) { + log.warn( + "update property type, propName={}, oldType={}", newProp.getName(), oldProp.getType()); + throw new SpgException(SpgMessageEnum.SCHEMA_CHANGE_PROPERTY_TYPE); + } + SPGTypeRef propRef = getTargetType(newProp.getType(), namespace, name2Type); + boolean isInherited = oldProp == null ? false : isInherited(oldProp); + Property property = + new Property( + new BasicInfo<>( + new PredicateIdentifier(newProp.getName()), + newProp.getNameZh(), + StringUtils.EMPTY), + advancedType.toRef(), + propRef, + isInherited, + new PropertyAdvancedConfig().setIndexType(getIndexType(newProp.getIndex()))); + buildEventProperty(advancedType, property); + property.getAdvancedConfig().setConstraint(newProp.getConstraint()); + if (oldProp == null) { + property.getAdvancedConfig().setLogicalRule(setLogicalRule(null, newProp.getRule())); + property.setAlterOperation( + advancedType instanceof ConceptType ? null : AlterOperationEnum.CREATE); + } else { + property + .getAdvancedConfig() + .setLogicalRule(setLogicalRule(oldProp.getRule(), newProp.getRule())); + deleteProps.remove(newProp.getName()); + property.setOntologyId(oldProp.getOntologyId()); + property.setAlterOperation(AlterOperationEnum.UPDATE); + } + propertyList.add(property); + } + Set builtInPropertyNames = new HashSet<>(); + if (advancedType instanceof ConceptType) { + builtInPropertyNames = BuiltInPropertyEnum.getBuiltInPropertyName(SPGTypeEnum.CONCEPT_TYPE); + } else if (advancedType instanceof EventType) { + builtInPropertyNames = BuiltInPropertyEnum.getBuiltInPropertyName(SPGTypeEnum.EVENT_TYPE); + } + for (String name : deleteProps) { + PropertyModel oldProp = oldPropMap.get(name); + SPGTypeRef propRef = getTargetType(oldProp.getType(), namespace, name2Type); + boolean isInherited = isInherited(oldProp); + Property property = + new Property( + new BasicInfo<>(new PredicateIdentifier(name), name, StringUtils.EMPTY), + advancedType.toRef(), + propRef, + isInherited, + new PropertyAdvancedConfig()); + if (!isInherited && !builtInPropertyNames.contains(name)) { + property.setAlterOperation(AlterOperationEnum.DELETE); + } + property.getAdvancedConfig().setLogicalRule(oldProp.getRule()); + property.setOntologyId(oldProp.getOntologyId()); + propertyList.add(property); + } + advancedType.setProperties(propertyList); + } + + /** + * update relation property + * + * @param oldEdge + * @param properties new edge properties + * @param relation + */ + private static void updateRelationProperty( + EdgeTypeModel oldEdge, List properties, Relation relation) { + if (CollectionUtils.isEmpty(properties) + && (oldEdge == null || CollectionUtils.isEmpty(oldEdge.getProperties()))) { + return; + } + List subProperties = Lists.newArrayList(); + relation.getAdvancedConfig().setSubProperties(subProperties); + Map oldsubPropMap = Maps.newHashMap(); + List oldSubProp = Lists.newArrayList(); + if (oldEdge != null && CollectionUtils.isNotEmpty(oldEdge.getProperties())) { + List oldProps = oldEdge.getProperties(); + for (PropertyModel oldProp : oldProps) { + oldsubPropMap.put(oldProp.getName(), oldProp); + oldSubProp.add(oldProp.getName()); + } + } + List deleteProps = new ArrayList<>(oldSubProp); + if (CollectionUtils.isNotEmpty(properties)) { + relationPropertyCreateOrUpdate( + properties, relation, oldsubPropMap, deleteProps, subProperties); + } + if (CollectionUtils.isEmpty(deleteProps)) { + return; + } + deleteProps.forEach( + name -> { + PropertyModel oldProp = oldsubPropMap.get(name); + SubProperty subProperty = + new SubProperty( + new BasicInfo<>( + new PredicateIdentifier(oldProp.getName()), + oldProp.getNameZh(), + StringUtils.EMPTY), + relation.toRef(), + new SPGTypeRef( + new BasicInfo<>( + new SPGTypeIdentifier(null, BasicTypeEnum.TEXT.getCode()), + BasicTypeEnum.TEXT.getDesc(), + BasicTypeEnum.TEXT.getDesc()), + SPGTypeEnum.BASIC_TYPE), + new PropertyAdvancedConfig()); + subProperty.setOntologyId(oldProp.getOntologyId()); + if (oldProp.getInherited() == null || !oldProp.getInherited()) { + subProperty.setAlterOperation(AlterOperationEnum.DELETE); + } + subProperties.add(subProperty); + }); + } + + /** + * update relation property + * + * @param properties + * @param relation + * @param oldsubPropMap + * @param deleteProps + * @param subProperties + */ + private static void relationPropertyCreateOrUpdate( + List properties, + Relation relation, + Map oldsubPropMap, + List deleteProps, + List subProperties) { + for (PropertyModel newSubProp : properties) { + BasicTypeEnum propertyType = getPropertyType(newSubProp.getType()); + SubProperty subProperty = + new SubProperty( + new BasicInfo<>( + new PredicateIdentifier(newSubProp.getName()), + newSubProp.getNameZh(), + StringUtils.EMPTY), + relation.toRef(), + new SPGTypeRef( + new BasicInfo<>( + new SPGTypeIdentifier(null, propertyType.getCode()), + propertyType.getDesc(), + propertyType.getDesc()), + SPGTypeEnum.BASIC_TYPE), + new PropertyAdvancedConfig()); + PropertyModel propertyModel = oldsubPropMap.get(newSubProp.getName()); + if (propertyModel == null) { + subProperty.setAlterOperation(AlterOperationEnum.CREATE); + } else { + deleteProps.remove(newSubProp.getName()); + subProperty.setOntologyId(propertyModel.getOntologyId()); + subProperty.setAlterOperation(AlterOperationEnum.UPDATE); + } + subProperties.add(subProperty); + } + } + + /** + * Node add or delete + * + * @param namespace + * @param nodes + * @param advancedType + * @param name2Type + * @param alterOperation + */ + private static void convertToAddOrDelete( + String namespace, + List nodes, + List advancedType, + Map name2Type, + AlterOperationEnum alterOperation) { + if (CollectionUtils.isEmpty(nodes)) { + return; + } + if (name2Type.isEmpty()) { + name2Type = + nodes.stream().collect(Collectors.toMap(NodeTypeModel::getName, NodeTypeModel::getType)); + } + for (NodeTypeModel oldType : nodes) { + SPGTypeIdentifier spgTypeIdentifier = new SPGTypeIdentifier(namespace, oldType.getName()); + BasicInfo basicInfo = + new BasicInfo<>(spgTypeIdentifier, oldType.getNameZh(), oldType.getDesc()); + ParentTypeInfo parentTypeInfo = ParentTypeInfo.THING; + if (StringUtils.isNotBlank(oldType.getParentName())) { + parentTypeInfo = + new ParentTypeInfo(null, null, SPGTypeIdentifier.parse(oldType.getParentName()), null); + } + SPGTypeAdvancedConfig spgTypeAdvancedConfig = new SPGTypeAdvancedConfig(); + List props = + CollectionUtils.isEmpty(oldType.getProperties()) + ? Lists.newArrayList() + : oldType.getProperties(); + List propertyList = Lists.newArrayList(); + BaseAdvancedType type; + AdvancedTypeEnum advancedTypeEnum = AdvancedTypeEnum.valueOf(oldType.getType()); + switch (advancedTypeEnum) { + case ENTITY_TYPE: + type = + new EntityType(basicInfo, parentTypeInfo, propertyList, null, spgTypeAdvancedConfig); + break; + case EVENT_TYPE: + type = + new EventType(basicInfo, parentTypeInfo, propertyList, null, spgTypeAdvancedConfig); + break; + case INDEX_TYPE: + type = + new IndexType(basicInfo, parentTypeInfo, propertyList, null, spgTypeAdvancedConfig); + break; + case CONCEPT_TYPE: + String hypernymPredicate = + StringUtils.isBlank(oldType.getHypernymPredicate()) + ? SystemPredicateEnum.IS_A.getName() + : oldType.getHypernymPredicate(); + type = + new ConceptType( + basicInfo, + parentTypeInfo, + propertyList, + null, + spgTypeAdvancedConfig, + new ConceptLayerConfig(hypernymPredicate, null), + null, + null); + break; + default: + throw new RuntimeException("unsupported advanced type: " + oldType.getType()); + } + type.setOntologyId(oldType.getOntologyId()); + type.setExtInfo(new SchemaExtInfo()); + // Meta-concepts are currently not supported for directly pulling edges or creating + // attributes; proceed directly to skipping. + if (type instanceof ConceptType) { + type.setAlterOperation(alterOperation); + advancedType.add(type); + continue; + } + for (PropertyModel prop : props) { + SPGTypeRef propRef = getTargetType(prop.getType(), namespace, name2Type); + boolean isInherited = isInherited(prop); + Property property = + new Property( + new BasicInfo<>( + new PredicateIdentifier(prop.getName()), prop.getNameZh(), StringUtils.EMPTY), + type.toRef(), + propRef, + isInherited, + new PropertyAdvancedConfig().setIndexType(getIndexType(prop.getIndex()))); + property.setOntologyId(prop.getOntologyId()); + property.getAdvancedConfig().setLogicalRule(setLogicalRule(null, prop.getRule())); + property.getAdvancedConfig().setConstraint(prop.getConstraint()); + property.setExtInfo(new SchemaExtInfo()); + buildEventProperty(type, property); + if (!isInherited) { + property.setAlterOperation(alterOperation); + } + propertyList.add(property); + } + if (CollectionUtils.isNotEmpty(oldType.getRelations())) { + List relations = Lists.newArrayList(); + List relationModels = oldType.getRelations(); + for (EdgeTypeModel relationModel : relationModels) { + PropertyAdvancedConfig propertyAdvancedConfig = new PropertyAdvancedConfig(); + boolean isInherited = + relationModel.getInherited() != null && relationModel.getInherited(); + SPGTypeRef propRef = getTargetType(relationModel.getTargetType(), namespace, name2Type); + Relation relation = + new Relation( + new BasicInfo<>( + new PredicateIdentifier(relationModel.getName()), + relationModel.getNameZh(), + relationModel.getDesc()), + type.toRef(), + propRef, + isInherited, + propertyAdvancedConfig, + false); + relation + .getAdvancedConfig() + .setLogicalRule(setLogicalRule(null, relationModel.getRule())); + updateRelationProperty(relationModel, relationModel.getProperties(), relation); + relation.setOntologyId(relationModel.getOntologyId()); + relation.setExtInfo(new SchemaExtInfo()); + if (!isInherited) { + relation.setAlterOperation(alterOperation); + } + relations.add(relation); + } + type.setRelations(relations); + } + type.setAlterOperation(alterOperation); + advancedType.add(type); + } + } + + private static void buildEventProperty(BaseAdvancedType advancedType, Property property) { + if (advancedType instanceof EventType) { + String name = property.getName(); + if (PropertyGroupEnum.SUBJECT.getNameEn().equalsIgnoreCase(name)) { + property.getAdvancedConfig().setPropertyGroup(PropertyGroupEnum.SUBJECT); + } + if (PropertyGroupEnum.OBJECT.getNameEn().equalsIgnoreCase(name)) { + property.getAdvancedConfig().setPropertyGroup(PropertyGroupEnum.OBJECT); + } + } + } + + private static SPGTypeRef getTargetType( + String targetType, String namespace, Map typeMap) { + SPGTypeRef spgTypeRef; + if (BasicTypeEnum.isBasicType(targetType)) { + BasicTypeEnum propertyType = getPropertyType(targetType); + spgTypeRef = + new SPGTypeRef( + new BasicInfo<>( + new SPGTypeIdentifier(null, propertyType.getCode()), + propertyType.getDesc(), + propertyType.getDesc()), + SPGTypeEnum.BASIC_TYPE); + } else if (targetType.startsWith("STD.")) { + spgTypeRef = + new SPGTypeRef( + new BasicInfo<>(SPGTypeIdentifier.parse(targetType)), SPGTypeEnum.STANDARD_TYPE); + } else { + SPGTypeIdentifier propSpgTypeIdentifier = new SPGTypeIdentifier(namespace, targetType); + String s = typeMap.get(targetType); + if (StringUtils.isBlank(s)) { + throw new RuntimeException("can not find type: " + targetType); + } + spgTypeRef = new SPGTypeRef(new BasicInfo<>(propSpgTypeIdentifier), SPGTypeEnum.toEnum(s)); + } + return spgTypeRef; + } + + private static LogicalRule setLogicalRule(LogicalRule oldRule, LogicalRule newRule) { + if (oldRule == null && newRule == null) { + return null; + } + if (oldRule != null && newRule == null) { + return null; + } + if (oldRule == null && newRule != null) { + return newRule; + } + if (oldRule != null && newRule != null) { + newRule.setCode(oldRule.getCode()); + return newRule; + } + return null; + } + + /** + * get index type by index type string + * + * @param index + * @return + */ + private static IndexTypeEnum getIndexType(String index) { + IndexTypeEnum indexTypeEnum = null; + if (StringUtils.isNotBlank(index)) { + indexTypeEnum = IndexTypeEnum.valueOf(index.trim()); + } + return indexTypeEnum; + } + + /** + * Determine if inheriting attributes. + * + * @param propertyModel + * @return + */ + private static boolean isInherited(PropertyModel propertyModel) { + return propertyModel.getInherited() != null && propertyModel.getInherited(); + } + + private static BasicTypeEnum getPropertyType(String type) { + if (StringUtils.isBlank(type)) { + return BasicTypeEnum.TEXT; + } + return BasicTypeEnum.getByCode(type.trim()); + } + + @Getter + public class SchemaChangeDTO { + private List addTypes; + private List updateTypes; + private List deleteTypes; + + public SchemaChangeDTO( + List addTypes, + List updateTypes, + List deleteTypes) { + this.addTypes = addTypes == null ? new ArrayList<>() : addTypes; + this.updateTypes = updateTypes == null ? new ArrayList<>() : updateTypes; + this.deleteTypes = deleteTypes == null ? new ArrayList<>() : deleteTypes; + } + } +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaModel.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaModel.java new file mode 100644 index 000000000..a4a3a08b2 --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaModel.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import java.util.List; +import lombok.Data; + +@Data +public class SchemaModel { + /** project namespace */ + private String namespace; + + /** node type list */ + private List nodeTypeModels; +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaModelConvertor.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaModelConvertor.java new file mode 100644 index 000000000..54bf9f200 --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaModelConvertor.java @@ -0,0 +1,124 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import com.antgroup.openspg.core.schema.model.predicate.Property; +import com.antgroup.openspg.core.schema.model.predicate.SubProperty; +import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; +import com.antgroup.openspg.core.schema.model.type.ConceptType; +import java.util.ArrayList; +import java.util.List; + +public class SchemaModelConvertor { + + /** + * Convert BaseAdvancedType to AdvancedTypeModel + * + * @param namespace + * @param baseAdvancedType + * @return + */ + public static NodeTypeModel convert2NodeTypeModel( + String namespace, BaseAdvancedType baseAdvancedType) { + String prefix = namespace + "."; + NodeTypeModel nodeTypeModel = new NodeTypeModel(); + nodeTypeModel.setOntologyId(baseAdvancedType.getOntologyId()); + nodeTypeModel.setName(baseAdvancedType.getName()); + nodeTypeModel.setName(baseAdvancedType.getName().replace(prefix, "")); + nodeTypeModel.setNameZh(baseAdvancedType.getBasicInfo().getNameZh()); + nodeTypeModel.setDesc(baseAdvancedType.getBasicInfo().getDesc()); + List propertyModels = getPropertyModels(prefix, baseAdvancedType); + nodeTypeModel.setProperties(propertyModels); + nodeTypeModel.setType(baseAdvancedType.getSpgTypeEnum().name()); + nodeTypeModel.setParentName( + baseAdvancedType.getParentTypeName().getNameEn().replace(prefix, "")); + if (baseAdvancedType instanceof ConceptType) { + ConceptType conceptType = (ConceptType) baseAdvancedType; + if (null != conceptType.getConceptLayerConfig()) { + nodeTypeModel.setHypernymPredicate( + conceptType.getConceptLayerConfig().getHypernymPredicate()); + } + } + // relation + List relations = new ArrayList<>(); + baseAdvancedType + .getRelations() + .forEach( + relation -> { + EdgeTypeModel edgeTypeModel = new EdgeTypeModel(); + edgeTypeModel.setOntologyId(relation.getOntologyId()); + edgeTypeModel.setName(relation.getName()); + edgeTypeModel.setInherited(relation.getInherited()); + edgeTypeModel.setSemanticRelation(relation.isSemanticRelation()); + edgeTypeModel.setNameZh(relation.getBasicInfo().getNameZh()); + edgeTypeModel.setDesc(relation.getBasicInfo().getDesc()); + edgeTypeModel.setSourceType( + relation.getSubjectTypeRef().getName().replace(prefix, "")); + edgeTypeModel.setTargetType( + relation.getObjectTypeRef().getName().replace(prefix, "")); + List edgeProperty = getPropertyModels(relation.getSubProperties()); + edgeTypeModel.setProperties(edgeProperty); + edgeTypeModel.setRule(relation.getLogicalRule()); + relations.add(edgeTypeModel); + }); + nodeTypeModel.setRelations(relations); + return nodeTypeModel; + } + + /** + * Property of attribute or relation's property + * + * @param subProperties + * @return + */ + private static List getPropertyModels(List subProperties) { + List propertyModels = new ArrayList<>(); + for (SubProperty subProperty : subProperties) { + PropertyModel propertyModel = new PropertyModel(); + propertyModel.setOntologyId(subProperty.getOntologyId()); + propertyModel.setName(subProperty.getName()); + propertyModel.setNameZh(subProperty.getBasicInfo().getNameZh()); + propertyModel.setType(subProperty.getObjectTypeRef().getName()); + if (null != subProperty.getAdvancedConfig() + && subProperty.getAdvancedConfig().getIndexType() != null) { + propertyModel.setIndex(subProperty.getAdvancedConfig().getIndexType().name()); + } + propertyModel.setConstraint(subProperty.getConstraint()); + propertyModels.add(propertyModel); + } + return propertyModels; + } + + private static List getPropertyModels( + String prefix, BaseAdvancedType baseAdvancedType) { + List propertyModels = new ArrayList<>(); + List properties = baseAdvancedType.getProperties(); + for (Property property : properties) { + PropertyModel propertyModel = new PropertyModel(); + propertyModel.setOntologyId(property.getOntologyId()); + propertyModel.setName(property.getName()); + propertyModel.setNameZh(property.getBasicInfo().getNameZh()); + propertyModel.setType(property.getObjectTypeRef().getName().replace(prefix, "")); + propertyModel.setInherited(property.getInherited()); + if (null != property.getAdvancedConfig() + && property.getAdvancedConfig().getIndexType() != null) { + propertyModel.setIndex(property.getAdvancedConfig().getIndexType().name()); + } + propertyModel.setRule(property.getLogicalRule()); + propertyModel.setConstraint(property.getConstraint()); + propertyModel.setSubProperties(getPropertyModels(property.getSubProperties())); + propertyModels.add(propertyModel); + } + return propertyModels; + } +} diff --git a/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaScriptTranslateUtil.java b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaScriptTranslateUtil.java new file mode 100644 index 000000000..40a6c43cf --- /dev/null +++ b/server/biz/schema/src/main/java/com/antgroup/openspg/server/biz/schema/model/SchemaScriptTranslateUtil.java @@ -0,0 +1,647 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.biz.schema.model; + +import com.antgroup.openspg.common.constants.SpgAppConstant; +import com.antgroup.openspg.common.util.enums.AdvancedTypeEnum; +import com.antgroup.openspg.core.schema.model.constraint.BaseConstraintItem; +import com.antgroup.openspg.core.schema.model.constraint.Constraint; +import com.antgroup.openspg.core.schema.model.constraint.ConstraintTypeEnum; +import com.antgroup.openspg.core.schema.model.constraint.EnumConstraint; +import com.antgroup.openspg.core.schema.model.constraint.MultiValConstraint; +import com.antgroup.openspg.core.schema.model.constraint.NotNullConstraint; +import com.antgroup.openspg.core.schema.model.constraint.RegularConstraint; +import com.antgroup.openspg.core.schema.model.predicate.IndexTypeEnum; +import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; +import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; +import com.antgroup.openspg.server.core.schema.service.type.model.BuiltInPropertyEnum; +import com.google.common.base.CharMatcher; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.MutableTriple; + +/** schema script translate util */ +public class SchemaScriptTranslateUtil { + + private static final String PROP = "PROP"; + private static final String EDGE = "EDGE"; + private static final String RULE = "RULE"; + private static final List RULE_KEYWORDS = + Lists.newArrayList("DEFINE", "STRUCTURE", "CONSTRAINT"); + + /** + * translate schema model to schema script + * + * @param schemaModel + * @return + */ + public static String translateSchema(SchemaModel schemaModel) { + StringBuilder sb = new StringBuilder(); + sb.append("namespace ").append(schemaModel.getNamespace()).append(SpgAppConstant.EMPTY_LINE); + if (CollectionUtils.isEmpty(schemaModel.getNodeTypeModels())) { + return sb.toString(); + } + for (NodeTypeModel nodeTypeModel : schemaModel.getNodeTypeModels()) { + AdvancedTypeEnum advancedTypeEnum = AdvancedTypeEnum.valueOf(nodeTypeModel.getType()); + sb.append(nodeTypeModel.getName()) + .append("(") + .append(nodeTypeModel.getNameZh()) + .append("): ") + .append(advancedTypeEnum.getCode()) + .append(SpgAppConstant.LINE_SEPARATOR); + if (StringUtils.isNotBlank(nodeTypeModel.getDesc())) { + setDeep(1, sb); + sb.append("desc:").append(nodeTypeModel.getDesc()).append(SpgAppConstant.LINE_SEPARATOR); + } + if (AdvancedTypeEnum.CONCEPT_TYPE.equals(advancedTypeEnum) + && StringUtils.isNotBlank(nodeTypeModel.getHypernymPredicate())) { + setDeep(1, sb); + sb.append("hypernymPredicate: ") + .append(nodeTypeModel.getHypernymPredicate()) + .append(SpgAppConstant.LINE_SEPARATOR) + .append(SpgAppConstant.LINE_SEPARATOR); + continue; + } + translateProperties(schemaModel.getNamespace(), nodeTypeModel, sb); + translateRelations(schemaModel.getNamespace(), nodeTypeModel, sb); + sb.append(SpgAppConstant.LINE_SEPARATOR); + } + return sb.toString(); + } + + private static void translateRelations( + String namespace, NodeTypeModel nodeTypeModel, StringBuilder sb) { + if (CollectionUtils.isEmpty(nodeTypeModel.getRelations())) { + return; + } + StringBuilder relationSb = new StringBuilder(); + for (EdgeTypeModel edgeTypeModel : nodeTypeModel.getRelations()) { + boolean semanticRelation = + edgeTypeModel.getSemanticRelation() != null && edgeTypeModel.getSemanticRelation(); + if (semanticRelation) { + continue; + } + setDeep(2, relationSb); + relationSb + .append(edgeTypeModel.getName()) + .append("(") + .append(edgeTypeModel.getNameZh()) + .append("): ") + .append(edgeTypeModel.getTargetType()) + .append(SpgAppConstant.LINE_SEPARATOR); + if (CollectionUtils.isNotEmpty(edgeTypeModel.getProperties())) { + setDeep(3, relationSb); + relationSb.append("properties:").append(SpgAppConstant.LINE_SEPARATOR); + for (PropertyModel property : edgeTypeModel.getProperties()) { + if (property.getInherited() != null && property.getInherited()) { + continue; + } + setDeep(4, relationSb); + relationSb + .append(property.getName()) + .append("(") + .append(property.getNameZh()) + .append("): ") + .append(property.getType()) + .append(SpgAppConstant.LINE_SEPARATOR); + if (StringUtils.isNotEmpty(property.getIndex())) { + setDeep(5, relationSb); + relationSb + .append("index:") + .append(property.getIndex()) + .append(SpgAppConstant.LINE_SEPARATOR); + } + } + } + if (edgeTypeModel.getRule() != null + && StringUtils.isNotBlank(edgeTypeModel.getRule().getContent())) { + translateRule(namespace, edgeTypeModel.getRule().getContent(), relationSb); + } + } + if (relationSb.length() > 0) { + setDeep(1, sb); + sb.append("relations:").append(SpgAppConstant.LINE_SEPARATOR).append(relationSb); + } + } + + private static void translateProperties( + String namespace, NodeTypeModel nodeTypeModel, StringBuilder sb) { + if (CollectionUtils.isEmpty(nodeTypeModel.getProperties())) { + return; + } + Set builtInPropertyNames = new HashSet<>(); + AdvancedTypeEnum advancedTypeEnum = AdvancedTypeEnum.valueOf(nodeTypeModel.getType()); + if (AdvancedTypeEnum.EVENT_TYPE.equals(advancedTypeEnum)) { + builtInPropertyNames = BuiltInPropertyEnum.getBuiltInPropertyName(SPGTypeEnum.EVENT_TYPE); + } + StringBuilder propertySb = new StringBuilder(); + for (PropertyModel property : nodeTypeModel.getProperties()) { + if ((property.getInherited() != null && property.getInherited()) + || builtInPropertyNames.contains(property.getName())) { + continue; + } + setDeep(2, propertySb); + propertySb + .append(property.getName()) + .append("(") + .append(property.getNameZh()) + .append("): ") + .append(property.getType()) + .append(SpgAppConstant.LINE_SEPARATOR); + if (StringUtils.isNotEmpty(property.getIndex())) { + setDeep(3, propertySb); + propertySb + .append("index:") + .append(IndexTypeEnum.toEnum(property.getIndex()).getScriptName()) + .append(SpgAppConstant.LINE_SEPARATOR); + } + if (property.getConstraint() != null + && CollectionUtils.isNotEmpty(property.getConstraint().getConstraintItems())) { + setDeep(3, propertySb); + propertySb.append("constraint:"); + List constraintItems = property.getConstraint().getConstraintItems(); + StringBuilder constraintBuilder = new StringBuilder(); + for (int i = 0; i < constraintItems.size(); i++) { + ConstraintTypeEnum constraintTypeEnum = constraintItems.get(i).getConstraintTypeEnum(); + switch (constraintTypeEnum) { + case MULTI_VALUE: + appendIfNotEmpty(constraintBuilder, ",").append("MultiValue"); + break; + case ENUM: + EnumConstraint enumConstraint = (EnumConstraint) constraintItems.get(i); + appendIfNotEmpty(constraintBuilder, ",") + .append("Enum=") + .append("\"") + .append(Joiner.on(",").join(enumConstraint.getEnumValues())) + .append("\""); + break; + case UNIQUE: + appendIfNotEmpty(constraintBuilder, ",").append("Unique"); + break; + case NOT_NULL: + appendIfNotEmpty(constraintBuilder, ",").append("NotNull"); + break; + case RANGE: + case REGULAR: + default: + break; + } + } + propertySb.append(constraintBuilder); + propertySb.append(SpgAppConstant.LINE_SEPARATOR); + } + if (property.getRule() != null && StringUtils.isNotBlank(property.getRule().getContent())) { + translateRule(namespace, property.getRule().getContent(), propertySb); + } + } + if (propertySb.length() > 0) { + setDeep(1, sb); + sb.append("properties:").append(SpgAppConstant.LINE_SEPARATOR).append(propertySb); + } + } + + private static void translateRule(String namespace, String rule, StringBuilder sb) { + setDeep(3, sb); + String ruleScript = rule.replace(String.format("%s.", namespace), ""); + sb.append("rule: [[").append(SpgAppConstant.LINE_SEPARATOR); + if (ruleScript.contains(SpgAppConstant.LINE_SEPARATOR)) { + List ruleLines = + Splitter.on(SpgAppConstant.LINE_SEPARATOR).omitEmptyStrings().splitToList(ruleScript); + for (int i = 0; i < ruleLines.size(); i++) { + String ruleLine = ruleLines.get(i); + if (indentCount(ruleLine) < 16) { + setDeep(startWithKeyWords(ruleLine) ? 4 : 5, sb); + } + sb.append(ruleLine).append(SpgAppConstant.LINE_SEPARATOR); + } + } else { + setDeep(4, sb); + sb.append(ruleScript).append(SpgAppConstant.LINE_SEPARATOR); + } + setDeep(4, sb); + sb.append("]]").append(SpgAppConstant.LINE_SEPARATOR); + } + + private static boolean startWithKeyWords(String ruleLine) { + boolean result = false; + for (String keyword : RULE_KEYWORDS) { + result = ruleLine.trim().toUpperCase().startsWith(keyword); + if (result) { + return result; + } + } + return result; + } + + /** + * complete rule nodeName + * + * @param namespace + * @param rule + * @return + */ + private static String completeRule(String namespace, String rule) { + Pattern pattern = + Pattern.compile("\\(([\\w\\s]*?:)(`?[\\w\\s.]+`?)/?[^)]*?\\)", Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(rule); + while (matcher.find()) { + String group0 = matcher.group(0); + String group1 = matcher.group(1); + String group = matcher.group(2); + if (group.contains(".") || group.toLowerCase().matches("(integer|text|float)")) { + continue; + } + String nodeName = "(" + group1 + namespace + "." + group + ")"; + rule = rule.replace(group0, nodeName); + } + return rule.trim(); + } + + private static void setDeep(int deep, StringBuilder sb) { + for (int i = 0; i < deep; i++) { + sb.append(SpgAppConstant.TAB_SEPARATOR); + } + } + + /** + * translate schema script to schema model + * + * @param script + * @return + */ + public static SchemaModel translateScript(String script) { + SchemaModel schemaModel = new SchemaModel(); + List lines = + Splitter.on(SpgAppConstant.LINE_SEPARATOR).omitEmptyStrings().splitToList(script.trim()); + if (CollectionUtils.isEmpty(lines)) { + throw new IllegalArgumentException("schema structure error"); + } + String firstLine = lines.get(0); + String namespace = null; + if (firstLine.contains("namespace")) { + namespace = firstLine.replace("namespace", "").trim(); + } + if (StringUtils.isBlank(namespace)) { + throw new IllegalArgumentException("schema miss namespace"); + } + schemaModel.setNamespace(namespace); + List nodeTypeList = Lists.newArrayList(); + List typeList = null; + for (int i = 1; i < lines.size(); i++) { + String line = lines.get(i); + if (StringUtils.isBlank(line) || line.startsWith("#")) { + continue; + } + int indentCount = indentCount(line); + boolean lastLine = i == lines.size() - 1; + if (indentCount == 0) { + if (CollectionUtils.isNotEmpty(typeList)) { + NodeTypeModel nodeTypeModel = toNodeTypeModel(namespace, typeList); + nodeTypeList.add(nodeTypeModel); + } + typeList = Lists.newArrayList(); + typeList.add(line); + } else { + typeList.add(line); + } + if (lastLine && CollectionUtils.isNotEmpty(typeList)) { + NodeTypeModel nodeTypeModel = toNodeTypeModel(namespace, typeList); + nodeTypeList.add(nodeTypeModel); + } + } + schemaModel.setNodeTypeModels(nodeTypeList); + return schemaModel; + } + + private static NodeTypeModel toNodeTypeModel(String namespace, List typeLineList) { + NodeTypeModel nodeTypeModel = new NodeTypeModel(); + String first = typeLineList.get(0); + MutableTriple type = getType(first); + nodeTypeModel.setName(type.getLeft()); + nodeTypeModel.setNameZh(type.getMiddle()); + nodeTypeModel.setType(AdvancedTypeEnum.toEnumByCode(type.getRight()).name()); + if (typeLineList.size() > 1) { + List propStrList = Lists.newArrayList(); + List relationsStrList = Lists.newArrayList(); + splitPopAndEdge(nodeTypeModel, typeLineList, propStrList, relationsStrList); + List properties = getPropertyModelList(namespace, propStrList); + List relations = getRelationList(namespace, relationsStrList); + nodeTypeModel.setProperties(properties); + nodeTypeModel.setRelations(relations); + } + return nodeTypeModel; + } + + private static List getRelationList( + String namespace, List relationsStrList) { + List relations = new ArrayList<>(); + int spaceCount = 0; + String currentStep = null; + List propStrLine = null; + List relationRuleStr = null; + EdgeTypeModel last = null; + Map> relationPropMap = Maps.newHashMap(); + Map> relationRuleMap = Maps.newHashMap(); + for (String line : relationsStrList) { + int currentLintCount = indentCount(line); + if ("properties:".equals(line.trim())) { + spaceCount = currentLintCount; + currentStep = PROP; + propStrLine = Lists.newArrayList(); + relationPropMap.put(last, propStrLine); + continue; + } + if (PROP.equals(currentStep) && spaceCount >= currentLintCount) { + currentStep = EDGE; + } + if (line.trim().startsWith("rule:")) { + spaceCount = currentLintCount; + currentStep = RULE; + relationRuleStr = Lists.newArrayList(); + relationRuleMap.put(last, relationRuleStr); + relationRuleStr.add(line.trim().replace("rule:", "")); + continue; + } + if (RULE.equals(currentStep) && spaceCount >= currentLintCount) { + currentStep = EDGE; + } + + if (PROP.equals(currentStep)) { + propStrLine.add(line.replaceAll(SpgAppConstant.TAB_SEPARATOR, "")); + } else if (RULE.equals(currentStep)) { + relationRuleStr.add(line.trim()); + } else if (line.contains("):")) { + MutableTriple propType = getType(line); + EdgeTypeModel edgeTypeModel = new EdgeTypeModel(); + edgeTypeModel.setName(propType.getLeft()); + edgeTypeModel.setNameZh(propType.getMiddle()); + edgeTypeModel.setTargetType(propType.getRight()); + relations.add(edgeTypeModel); + last = edgeTypeModel; + } + } + for (Entry> entry : relationPropMap.entrySet()) { + if (CollectionUtils.isEmpty(entry.getValue())) { + continue; + } + EdgeTypeModel edgeTypeModel = entry.getKey(); + List value = entry.getValue(); + List propertyModelList = getPropertyModelList(namespace, value); + edgeTypeModel.setProperties(propertyModelList); + } + for (Entry> entry : relationRuleMap.entrySet()) { + if (CollectionUtils.isEmpty(entry.getValue())) { + continue; + } + EdgeTypeModel edgeTypeModel = entry.getKey(); + List value = entry.getValue(); + String rule = Joiner.on(SpgAppConstant.LINE_SEPARATOR).join(value).trim(); + rule = completeRule(namespace, rule); + if (rule.startsWith("[[")) { + rule = rule.replace("[[", ""); + } + if (rule.endsWith("]]")) { + rule = rule.replace("]]", ""); + } + edgeTypeModel.setRule(new LogicalRule(null, null, rule)); + } + return relations; + } + + /** + * get Node property + * + * @param propStrList + * @return + */ + private static List getPropertyModelList( + String namespace, List propStrList) { + List result = new ArrayList<>(); + PropertyModel last = null; + int spaceCount = 0; + String currentStep = null; + List propStrLine = null; + List ruleStrList = null; + Map> subPropMap = Maps.newHashMap(); + Map> porpRuleMap = Maps.newHashMap(); + for (String line : propStrList) { + int currentLintCount = indentCount(line); + if ("properties:".equals(line.trim())) { + spaceCount = currentLintCount; + currentStep = PROP; + propStrLine = Lists.newArrayList(); + subPropMap.put(last, propStrLine); + continue; + } + if (PROP.equals(currentStep) && spaceCount >= currentLintCount) { + currentStep = null; + } + if (line.trim().startsWith("rule:")) { + spaceCount = currentLintCount; + currentStep = RULE; + ruleStrList = Lists.newArrayList(); + porpRuleMap.put(last, ruleStrList); + ruleStrList.add(line.trim().replace("rule:", "")); + continue; + } + if (RULE.equals(currentStep) && spaceCount >= currentLintCount) { + currentStep = EDGE; + } + + if (PROP.equals(currentStep)) { + propStrLine.add(line.replaceAll(SpgAppConstant.TAB_SEPARATOR, "")); + } else if (RULE.equals(currentStep)) { + ruleStrList.add(line.trim()); + } else if (line.contains("):")) { + MutableTriple propType = getType(line); + PropertyModel propertyModel = new PropertyModel(); + propertyModel.setName(propType.getLeft()); + propertyModel.setNameZh(propType.getMiddle()); + propertyModel.setType(propType.getRight()); + last = propertyModel; + result.add(propertyModel); + } + if (line.contains("index:") && last != null) { + String scriptName = Splitter.on(":").trimResults().splitToList(line).get(1); + last.setIndex(IndexTypeEnum.getByScriptName(scriptName).name()); + } + if (line.contains("constraint:") && last != null) { + String constraintStr = Splitter.on(":").trimResults().splitToList(line).get(1); + Constraint constraint = new Constraint(); + List list = new ArrayList<>(); + parseConstraintForProperty(constraintStr, list); + constraint.setConstraintItems(list); + last.setConstraint(constraint); + } + } + for (Entry> entry : subPropMap.entrySet()) { + if (CollectionUtils.isEmpty(entry.getValue())) { + continue; + } + PropertyModel edgeTypeModel = entry.getKey(); + List value = entry.getValue(); + List propertyModelList = getPropertyModelList(namespace, value); + edgeTypeModel.setSubProperties(propertyModelList); + } + for (Entry> entry : porpRuleMap.entrySet()) { + if (CollectionUtils.isEmpty(entry.getValue())) { + continue; + } + PropertyModel propertyModel = entry.getKey(); + List value = entry.getValue(); + String rule = Joiner.on(SpgAppConstant.LINE_SEPARATOR).join(value).trim(); + rule = completeRule(namespace, rule); + if (rule.startsWith("[[")) { + rule = rule.replace("[[", ""); + } + if (rule.endsWith("]]")) { + rule = rule.replace("]]", ""); + } + propertyModel.setRule(new LogicalRule(null, null, rule)); + } + return result; + } + + /** parse constraint for property */ + private static void parseConstraintForProperty( + String expression, List constraintItems) { + if (StringUtils.isBlank(expression)) { + return; + } + Pattern pattern = + Pattern.compile("(Enum|Regular)\\s*?=\\s*?\"([^\"]+)\"", Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(expression); + while (matcher.find()) { + String constraintType = matcher.group(1).toLowerCase(); + String constraintValue = matcher.group(2).trim(); + if ("enum".equals(constraintType)) { + String[] enumValues = constraintValue.split(","); + List stripEnumValues = new ArrayList<>(); + for (String ev : enumValues) { + stripEnumValues.add(ev.trim()); + } + constraintItems.add(new EnumConstraint(stripEnumValues)); + } else if ("regular".equals(constraintType)) { + constraintItems.add(new RegularConstraint(constraintValue)); + } + expression = expression.replaceFirst(matcher.group(), ""); + } + String[] array = expression.split(","); + for (String cons : array) { + cons = cons.trim(); + if ("multivalue".equalsIgnoreCase(cons)) { + constraintItems.add(new MultiValConstraint()); + } else if ("notnull".equalsIgnoreCase(cons)) { + constraintItems.add(new NotNullConstraint()); + } + } + } + + private static void splitPopAndEdge( + NodeTypeModel nodeTypeModel, + List typeLineList, + List propStrList, + List relationsStrList) { + int spaceCount = 0; + String currentStep = null; + for (int i = 1; i < typeLineList.size(); i++) { + String line = typeLineList.get(i); + int currentLintCount = indentCount(line); + if ("properties:".equals(line.trim()) && currentStep == null) { + spaceCount = currentLintCount; + currentStep = PROP; + continue; + } + if ("relations:".equals(line.trim())) { + spaceCount = currentLintCount; + currentStep = EDGE; + continue; + } + if (line.trim().startsWith("hypernymPredicate:")) { + spaceCount = currentLintCount; + currentStep = null; + String constraintStr = Splitter.on(":").trimResults().splitToList(line).get(1); + nodeTypeModel.setHypernymPredicate(constraintStr); + continue; + } + if (line.trim().startsWith("desc:")) { + spaceCount = currentLintCount; + currentStep = null; + String constraintStr = Splitter.on(":").trimResults().splitToList(line).get(1); + nodeTypeModel.setDesc(constraintStr); + continue; + } + if (PROP.equals(currentStep) && spaceCount < currentLintCount) { + propStrList.add(line); + } + if (EDGE.equals(currentStep) && spaceCount < indentCount(line)) { + relationsStrList.add(line); + } + } + } + + /** + * Count the number of spaces at the beginning of a string. Tabs are assumed to be equivalent to 4 + * spaces by default. + * + * @param input + * @return + */ + private static int indentCount(String input) { + int count = 0; + for (char c : input.toCharArray()) { + if (c == '\t') { + count = count + 4; + } else if (c == ' ') { + count++; + } else { + break; + } + } + return count; + } + + /** + * Appends the specified string to the given StringBuilder if it is not empty. + * + * @param builder The StringBuilder to append to. + * @param str The string to append. + */ + public static StringBuilder appendIfNotEmpty(StringBuilder builder, String str) { + if (builder.length() > 0) { + builder.append(str); + } + return builder; + } + + private static MutableTriple getType(String s) { + List list = + Splitter.on(CharMatcher.anyOf("():")).omitEmptyStrings().trimResults().splitToList(s); + if (CollectionUtils.isEmpty(list)) { + return null; + } + return MutableTriple.of(list.get(0), list.get(1), list.get(2)); + } +} diff --git a/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/SearchManager.java b/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/SearchManager.java index 5b5fd430c..3107bfed6 100644 --- a/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/SearchManager.java +++ b/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/SearchManager.java @@ -14,6 +14,7 @@ package com.antgroup.openspg.server.biz.service; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.idx.record.IdxRecord; +import com.antgroup.openspg.server.api.facade.dto.service.request.CustomSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.SPGTypeSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.TextSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.VectorSearchRequest; @@ -26,4 +27,6 @@ public interface SearchManager { List textSearch(TextSearchRequest request); List vectorSearch(VectorSearchRequest request); + + List customSearch(CustomSearchRequest request); } diff --git a/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/impl/SearchManagerImpl.java b/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/impl/SearchManagerImpl.java index 88c16a3a4..847d42e8e 100644 --- a/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/impl/SearchManagerImpl.java +++ b/server/biz/service/src/main/java/com/antgroup/openspg/server/biz/service/impl/SearchManagerImpl.java @@ -20,6 +20,7 @@ import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.*; import com.antgroup.openspg.common.constants.SpgAppConstant; import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.api.facade.dto.service.request.CustomSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.SPGTypeSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.TextSearchRequest; import com.antgroup.openspg.server.api.facade.dto.service.request.VectorSearchRequest; @@ -103,6 +104,16 @@ public List vectorSearch(VectorSearchRequest request) { return searchEngineClient.search(searchRequest); } + @Override + public List customSearch(CustomSearchRequest request) { + String searchEngineUrl = projectManager.getGraphStoreUrl(request.getProjectId()); + IdxDataQueryService searchEngineClient = getSearchEngineClient(searchEngineUrl); + String customQuery = request.getCustomQuery(); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setQuery(new CustomSearchQuery(customQuery)); + return searchEngineClient.search(searchRequest); + } + private IdxDataQueryService getSearchEngineClient(String searchEngineUrl) { return SearchEngineClientDriverManager.getClient(searchEngineUrl); } diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/CommonConstants.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/CommonConstants.java index 09a981a6f..9eb59e76e 100644 --- a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/CommonConstants.java +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/CommonConstants.java @@ -20,6 +20,7 @@ public class CommonConstants { public static final String VECTORIZER = "vectorizer"; public static final String VECTOR_DIMENSIONS = "vector_dimensions"; public static final String LLM = "llm"; - public static final String LLM_SELECT = "llm_select"; + public static final String CHAT = "chat"; public static final String PROMPT = "prompt"; + public static final String TASK_ID = "task_id"; } diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/app/App.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/app/App.java new file mode 100644 index 000000000..1fd985af1 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/app/App.java @@ -0,0 +1,66 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.common.model.app; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.common.model.base.BaseModel; +import java.util.Date; +import lombok.Data; + +@Data +public class App extends BaseModel { + private Long id; + private String name; + private String logo; + private String description; + private JSONObject config; + private String userNo; + private String accessToken; + // account token status + private Integer status; + private Date appDeployTime; + private String alias; + + public App() {} + + public App( + Long id, + String name, + String logo, + String description, + JSONObject config, + String userNo, + String alias) { + this.id = id; + this.name = name; + this.logo = logo; + this.description = description; + this.config = config; + this.userNo = userNo; + this.alias = alias; + } +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/bulider/BuilderJob.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/bulider/BuilderJob.java index 4eecadba4..5eb8495cb 100644 --- a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/bulider/BuilderJob.java +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/bulider/BuilderJob.java @@ -44,4 +44,5 @@ public class BuilderJob extends BaseModel { private String lifeCycle; private String action; private String dependence; + private String retrievals; } diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/config/Config.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/config/Config.java index 74884a57e..0c1b81b7a 100644 --- a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/config/Config.java +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/config/Config.java @@ -47,7 +47,7 @@ public class Config extends BaseModel { private Integer status; /** config json */ - private String config; + private Object config; /** version description */ private String description; @@ -58,13 +58,9 @@ public class Config extends BaseModel { /** resource type */ private String resourceType; - private boolean showProfilePicture = false; - - private boolean showUserConfig = false; - public Config() {} - public Config(Long id, String configName, String configId, String config, String resourceType) { + public Config(Long id, String configName, String configId, Object config, String resourceType) { this.id = id; this.configName = configName; this.configId = configId; diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/Feedback.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/Feedback.java new file mode 100644 index 000000000..fdbecd261 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/Feedback.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.feedback; + +import com.alibaba.fastjson.JSONObject; +import java.util.Date; +import lombok.Data; + +@Data +public class Feedback { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String moduleType; + private String oneCategory; + private String twoCategory; + private String threeCategory; + private String fourCategory; + private String fiveCategory; + private String reactionType; + private JSONObject reason; + private String userNo; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/FeedbackQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/FeedbackQuery.java new file mode 100644 index 000000000..1f1e235ce --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/FeedbackQuery.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.feedback; + +import lombok.Data; + +@Data +public class FeedbackQuery { + + private Long id; + private String moduleType; + private String oneCategory; + private String twoCategory; + private String threeCategory; + private String fourCategory; + private String fiveCategory; + private String reactionType; + private String sort; + private String order; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/FeedbackStatistics.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/FeedbackStatistics.java new file mode 100644 index 000000000..23636aade --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/feedback/FeedbackStatistics.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.feedback; + +import lombok.Data; + +@Data +public class FeedbackStatistics { + + private String oneCategory; + + private Long upNum = 0L; + + private Long downNum = 0L; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetail.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetail.java new file mode 100644 index 000000000..73460c0de --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetail.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.modeldetail; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.common.model.base.BaseModel; +import java.util.Date; +import lombok.Data; + +@Data +public class ModelDetail extends BaseModel { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String provider; + private String type; + private String name; + private String description; + private JSONObject params; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetailDTO.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetailDTO.java new file mode 100644 index 000000000..30d697e25 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetailDTO.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.modeldetail; + +import com.antgroup.openspg.server.common.model.base.BaseModel; +import lombok.Data; + +@Data +public class ModelDetailDTO extends BaseModel { + + /** model name */ + private String name; + + /** model type */ + private String type; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetailQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetailQuery.java new file mode 100644 index 000000000..f992b419d --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/modeldetail/ModelDetailQuery.java @@ -0,0 +1,23 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.modeldetail; + +import lombok.Data; + +@Data +public class ModelDetailQuery extends ModelDetail { + + private String sort; + + private String order; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/project/Project.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/project/Project.java index 5628cb0b3..7d09aefa1 100644 --- a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/project/Project.java +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/project/Project.java @@ -35,20 +35,32 @@ public class Project extends BaseModel { /** The namespace that isolate entity from different project. */ private final String namespace; + /** The visibility of the project */ + private String visibility; + /** The tenant id that project belong to. */ private final Long tenantId; /** Base configuration for the project dimension. */ private final String config; + private String tag; + public Project( - Long id, String name, String description, String namespace, Long tenantId, String config) { + Long id, + String name, + String description, + String namespace, + Long tenantId, + String config, + String tag) { this.id = id; this.name = name; this.description = description; this.namespace = namespace; this.tenantId = tenantId; this.config = config; + this.tag = tag; } public void setId(Long id) { @@ -78,4 +90,20 @@ public String getNamespace() { public String getConfig() { return config; } + + public String getVisibility() { + return visibility; + } + + public void setVisibility(String visibility) { + this.visibility = visibility; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } } diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/provider/ModelProvider.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/provider/ModelProvider.java new file mode 100644 index 000000000..bb210c74a --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/provider/ModelProvider.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.provider; + +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.server.common.model.base.BaseModel; +import java.util.Date; +import java.util.List; +import lombok.Data; + +@Data +public class ModelProvider extends BaseModel { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String name; + private String provider; + private String status; + private String pageMode; + private List modelType; + private String logo; + private String tags; + private JSONArray params; + + public static ModelProvider modelProvider(ModelProvider provider) { + ModelProvider modelProvider = new ModelProvider(); + modelProvider.setName(provider.getName()); + modelProvider.setProvider(provider.getProvider()); + modelProvider.setPageMode(provider.getPageMode()); + modelProvider.setModelType(provider.getModelType()); + modelProvider.setLogo(provider.getLogo()); + modelProvider.setTags(provider.getTags()); + modelProvider.setParams(provider.getParams()); + return modelProvider; + } +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/provider/ModelProviderQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/provider/ModelProviderQuery.java new file mode 100644 index 000000000..0bb18ae3c --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/provider/ModelProviderQuery.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.provider; + +import java.util.Date; +import lombok.Data; + +@Data +public class ModelProviderQuery { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String updateUser; + private String createUser; + private String name; + private String provider; + private String status; + private String pageMode; + private String modelType; + + private String sort; + + private String order; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/providerparam/ProviderParam.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/providerparam/ProviderParam.java new file mode 100644 index 000000000..743c0684e --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/providerparam/ProviderParam.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.providerparam; + +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.server.common.model.base.BaseModel; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import java.util.Date; +import java.util.List; +import lombok.Data; + +@Data +public class ProviderParam extends BaseModel { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String provider; + private String modelType; + private String modelName; + private JSONArray params; + private List model; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/providerparam/ProviderParamQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/providerparam/ProviderParamQuery.java new file mode 100644 index 000000000..b8d2f43e0 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/providerparam/ProviderParamQuery.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.providerparam; + +import lombok.Data; + +@Data +public class ProviderParamQuery extends ProviderParam { + private String sort; + + private String order; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefInfo.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefInfo.java new file mode 100644 index 000000000..e5d775339 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefInfo.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.common.model.ref; + +import com.antgroup.openspg.server.common.model.base.BaseModel; +import java.util.Date; +import lombok.Data; + +@Data +public class RefInfo extends BaseModel { + + private Long id; + + private Date gmtCreate; + + private Date gmtModified; + + private String name; + + private String refId; + + private String refType; + + private String refedId; + + private String refedType; + + private Integer status; + + private String config; + + public RefInfo() {} + + public RefInfo( + String name, String refId, String refType, String refedId, String refedType, Integer status) { + this.name = name; + this.refId = refId; + this.refType = refType; + this.refedId = refedId; + this.refedType = refedType; + this.status = status; + } +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefTypeEnum.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefTypeEnum.java new file mode 100644 index 000000000..e4b4694fc --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefTypeEnum.java @@ -0,0 +1,19 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.common.model.ref; + +public enum RefTypeEnum { + KNOWLEDGE_BASE, + APP, +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefedTypeEnum.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefedTypeEnum.java new file mode 100644 index 000000000..a3a64e6e5 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/ref/RefedTypeEnum.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.common.model.ref; + +public enum RefedTypeEnum { + LLM, + EMBEDDING, + ACCESS_TOKEN, + KNOWLEDGE_BASE, + API_KEY +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/retrieval/Retrieval.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/retrieval/Retrieval.java new file mode 100644 index 000000000..8cc28f445 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/retrieval/Retrieval.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.retrieval; + +import com.antgroup.openspg.server.common.model.base.BaseModel; +import java.util.Date; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Retrieval extends BaseModel { + + private static final long serialVersionUID = -1948526461818668802L; + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String updateUser; + private String createUser; + private String type; + private String status; + private String isDefault; + private String name; + private String chineseName; + private String schemaDesc; + private String scenariosDesc; + private String costDesc; + private String methodDesc; + private String extractorDesc; + private String retrieverDesc; + private String modulePath; + private String className; + private String method; + private String extension; + private String config; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/retrieval/RetrievalQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/retrieval/RetrievalQuery.java new file mode 100644 index 000000000..7a203052e --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/retrieval/RetrievalQuery.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.retrieval; + +import java.util.List; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class RetrievalQuery extends Retrieval { + + private static final long serialVersionUID = 5956770665095065648L; + + private List ids; + + private Integer pageNo; + + private Integer pageSize; + + private String sort; + + private String order; + + private String keyword; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/Statistics.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/Statistics.java new file mode 100644 index 000000000..7969fd43f --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/Statistics.java @@ -0,0 +1,52 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.statistics; + +import java.util.Date; +import lombok.Data; + +@Data +public class Statistics { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String resourceTag; + private String resourceId; + private String statisticsType; + private String statisticsDate; + private Long num; + + public Statistics() {} + + public Statistics( + Date date, + String userNo, + String resourceTag, + String resourceId, + String statisticsType, + String statisticsDate, + Long num) { + this.gmtCreate = date; + this.gmtModified = date; + this.modifier = userNo; + this.creator = userNo; + this.resourceTag = resourceTag; + this.resourceId = resourceId; + this.statisticsType = statisticsType; + this.statisticsDate = statisticsDate; + this.num = num; + } +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsInfo.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsInfo.java new file mode 100644 index 000000000..1afb8ea24 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsInfo.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.statistics; + +import lombok.Data; + +@Data +public class StatisticsInfo { + + private Long userNum = 0L; + private Long normalChatNum = 0L; + private Long debugChatNum = 0L; + private Long apiChatNum = 0L; + private Long chatNum = 0L; + private Long tokens = 0L; + private Long upNum = 0L; + private Long downNum = 0L; + private String statisticsDate; + + public StatisticsInfo() {} + + public StatisticsInfo(String statisticsDate) { + this.statisticsDate = statisticsDate; + } + + public StatisticsInfo( + Long userNum, + Long normalChatNum, + Long debugChatNum, + Long apiChatNum, + Long chatNum, + Long tokens, + Long upNum, + Long downNum) { + this.userNum = userNum; + this.normalChatNum = normalChatNum; + this.debugChatNum = debugChatNum; + this.apiChatNum = apiChatNum; + this.chatNum = chatNum; + this.tokens = tokens; + this.upNum = upNum; + this.downNum = downNum; + } +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsQuery.java new file mode 100644 index 000000000..6180123f0 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsQuery.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.statistics; + +import lombok.Data; + +@Data +public class StatisticsQuery { + + private Long id; + private String resourceTag; + private String resourceId; + private String statisticsType; + private String statisticsDate; + private Long num; + private String startTime; + private String endTime; + + private String sort; + private String order; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsTotal.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsTotal.java new file mode 100644 index 000000000..030810255 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/statistics/StatisticsTotal.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.statistics; + +import lombok.Data; + +@Data +public class StatisticsTotal { + + private Long userTotal = 0L; + private Long chatTotal = 0L; + private Long tokensTotal = 0L; + private Long upTotal = 0L; + private Long downTotal = 0L; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModel.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModel.java new file mode 100644 index 000000000..bf3debb7e --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModel.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.usermodel; + +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.server.common.model.base.BaseModel; +import java.util.Date; +import lombok.Data; + +@Data +public class UserModel extends BaseModel { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String userNo; + private String instanceId; + private String visibility; + private String provider; + private String name; + private String config; + private JSONArray modelList; + + public UserModel() {} + + public UserModel( + String instanceId, + String visibility, + String provider, + String name, + String userNo, + String modifier) { + this.instanceId = instanceId; + this.visibility = visibility; + this.provider = provider; + this.name = name; + this.userNo = userNo; + this.modifier = modifier; + } +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModelDTO.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModelDTO.java new file mode 100644 index 000000000..0bb251873 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModelDTO.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.usermodel; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.common.model.base.BaseModel; +import lombok.Data; + +@Data +public class UserModelDTO extends BaseModel { + + private Long id; + private String provider; + private String visibility; + private String name; + private JSONObject config; + private String userNo; + private String newName; + private JSONObject customize; +} diff --git a/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModelQuery.java b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModelQuery.java new file mode 100644 index 000000000..6a11002a9 --- /dev/null +++ b/server/common/model/src/main/java/com/antgroup/openspg/server/common/model/usermodel/UserModelQuery.java @@ -0,0 +1,23 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.model.usermodel; + +import lombok.Data; + +@Data +public class UserModelQuery extends UserModel { + + private String sort; + + private String order; +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/account/impl/AccountServiceDefaultImpl.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/account/impl/AccountServiceDefaultImpl.java index 3a323832c..6c8fca122 100644 --- a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/account/impl/AccountServiceDefaultImpl.java +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/account/impl/AccountServiceDefaultImpl.java @@ -10,7 +10,6 @@ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. */ - package com.antgroup.openspg.server.common.service.account.impl; import com.antgroup.openspg.server.api.facade.Paged; diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/app/AppRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/app/AppRepository.java new file mode 100644 index 000000000..b5a91dc56 --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/app/AppRepository.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.app; + +import com.antgroup.openspg.server.api.facade.dto.common.request.AppRequest; +import com.antgroup.openspg.server.common.model.app.App; +import java.util.List; + +public interface AppRepository { + Integer save(App app); + + Integer update(App app); + + App queryById(Long id); + + List queryPage(AppRequest request, int start, int size); + + Long selectCountByCondition(AppRequest request); + + Integer deleteById(Long id); + + App queryByName(String name); + + List queryByCondition(AppRequest request); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/config/DefaultValue.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/config/DefaultValue.java index 9754ed5a0..f6871d3c7 100644 --- a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/config/DefaultValue.java +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/config/DefaultValue.java @@ -32,6 +32,9 @@ public class DefaultValue { @Value("${cloudext.objectstorage.url:}") private String objectStorageUrl; + @Value("${cloudext.objectstorage.show.url:}") + private String objectStorageShowUrl; + @Value("${cloudext.computingengine.url:}") private String computingEngineUrl; @@ -41,6 +44,9 @@ public class DefaultValue { @Value("${builder.model.execute.num:20}") private Integer modelExecuteNum; + @Value("${builder.subgraph.batch.max:100}") + private Integer subGraphBatchMax; + @Value("${python.exec:}") private String pythonExec; @@ -58,4 +64,7 @@ public class DefaultValue { @Value("${yuque.api.url:}") private String yuQueApiUrl; + + @Value("${web_request_url:}") + private String webRequestUrl; } diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/modeldetail/ModelDetailRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/modeldetail/ModelDetailRepository.java new file mode 100644 index 000000000..a3faf806b --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/modeldetail/ModelDetailRepository.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.modeldetail; + +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetail; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import java.util.List; + +public interface ModelDetailRepository { + + /** insert model provider */ + Long insert(ModelDetail record); + + /** delete by id */ + int deleteById(Long id); + + /** update model provider */ + Long update(ModelDetail record); + + /** get by id */ + ModelDetail getById(Long id); + + /** query by condition */ + List query(ModelDetailQuery record); + + /** query by condition */ + List queryDTO(ModelDetailQuery record); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/permission/PermissionRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/permission/PermissionRepository.java index ea9e6d0b5..cd6d8dc66 100644 --- a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/permission/PermissionRepository.java +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/permission/PermissionRepository.java @@ -13,7 +13,6 @@ package com.antgroup.openspg.server.common.service.permission; -import com.antgroup.openspg.server.api.facade.Paged; import com.antgroup.openspg.server.common.model.permission.Permission; import java.util.List; @@ -36,17 +35,6 @@ public interface PermissionRepository { */ Integer update(Permission permission); - /** - * query all permission by resourceId - * - * @param resourceId - * @param resourceTag - * @param page - * @param pageSize - * @return - */ - List query(Long resourceId, String resourceTag, Integer page, Integer pageSize); - /** * query by roleId nad the part of userNo * @@ -59,31 +47,7 @@ public interface PermissionRepository { * @return */ List queryByUserNoAndRoleId( - String userNo, - Long roleId, - Long resourceId, - String resourceTag, - Integer page, - Integer pageSize); - - /** - * query page - * - * @param userNo - * @param roleId - * @param resourceId - * @param resourceTag - * @param page - * @param pageSize - * @return - */ - Paged queryPage( - String userNo, - Long roleId, - Long resourceId, - String resourceTag, - Integer page, - Integer pageSize); + String userNo, Long roleId, Long resourceId, String resourceTag, Long page, Long pageSize); /** * delete permission @@ -130,4 +94,36 @@ List getPermissionByUserRolesAndId( * @return */ Permission selectByPrimaryKey(Long id); + + /** + * select page + * + * @param userNo + * @param roleId + * @param resourceTag + * @return + */ + List selectLikeByUserNoAndRoleId( + String userNo, Long roleId, Long resourceId, String resourceTag, Long start, Long size); + + /** + * the count of selectLikeByUserNoAndRoleId + * + * @param userNo + * @param roleId + * @param resourceId + * @param resourceTag + * @return + */ + long selectLikeCountByUserNoAndRoleId( + String userNo, Long roleId, Long resourceId, String resourceTag); + + /** + * Delete Records Corresponding to the Resource + * + * @param resourceId + * @param resourceTag + * @return + */ + int deleteByResourceId(Long resourceId, String resourceTag); } diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/ProjectRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/ProjectRepository.java index f88daf55c..bd0b7716b 100644 --- a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/ProjectRepository.java +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/ProjectRepository.java @@ -13,7 +13,6 @@ package com.antgroup.openspg.server.common.service.project; -import com.antgroup.openspg.server.api.facade.Paged; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest; import com.antgroup.openspg.server.common.model.project.Project; import java.util.List; @@ -30,7 +29,9 @@ public interface ProjectRepository { Integer deleteById(Long projectId); - Paged queryPaged(ProjectQueryRequest request, int start, int size); + List queryPageData(ProjectQueryRequest request, int start, int size); + + Long queryPageCount(ProjectQueryRequest request); Project queryByNamespace(String namespace); } diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/impl/ProjectServiceImpl.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/impl/ProjectServiceImpl.java index db7f1d93c..a10272746 100644 --- a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/impl/ProjectServiceImpl.java +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/project/impl/ProjectServiceImpl.java @@ -16,11 +16,15 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.antgroup.openspg.cloudext.impl.graphstore.neo4j.Neo4jConstants; +import com.antgroup.openspg.common.constants.BuilderConstant; +import com.antgroup.openspg.common.constants.SpgAppConstant; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.server.common.model.CommonConstants; import com.antgroup.openspg.server.common.model.project.Project; import com.antgroup.openspg.server.common.service.project.ProjectRepository; import com.antgroup.openspg.server.common.service.project.ProjectService; +import com.antgroup.openspg.server.common.service.usermodel.UserModelRepository; +import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -32,17 +36,52 @@ public class ProjectServiceImpl implements ProjectService { @Autowired private ProjectRepository projectRepository; + @Autowired private UserModelRepository userModelRepository; + @Value("${cloudext.graphstore.url:}") private String url; @Override public Project queryById(Long projectId) { - return projectRepository.queryById(projectId); + Project project = projectRepository.queryById(projectId); + if (project == null) { + return null; + } + if (StringUtils.isBlank(project.getConfig())) { + return project; + } + JSONObject projectConfig = JSON.parseObject(project.getConfig()); + if (projectConfig != null && projectConfig.containsKey(CommonConstants.VECTORIZER)) { + JSONObject vectorizer = projectConfig.getJSONObject(CommonConstants.VECTORIZER); + String modelId = vectorizer.getString(SpgAppConstant.MODEL_ID); + JSONObject llmInfo = userModelRepository.getByModelId(modelId); + if (llmInfo != null) { + for (Map.Entry entry : llmInfo.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + vectorizer.put(key, value); + } + projectConfig.put(BuilderConstant.VECTORIZER, vectorizer); + } + project = + new Project( + project.getId(), + project.getName(), + project.getDescription(), + project.getNamespace(), + project.getTenantId(), + projectConfig.toJSONString(), + project.getTag()); + } + return project; } @Override public String getGraphStoreUrl(Long projectId) { Project project = projectRepository.queryById(projectId); + if (project == null) { + return this.url; + } String url = this.url; String user = null; String password = null; diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/provider/ModelProviderRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/provider/ModelProviderRepository.java new file mode 100644 index 000000000..483cd29ba --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/provider/ModelProviderRepository.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.provider; + +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import com.antgroup.openspg.server.common.model.provider.ModelProviderQuery; +import java.util.List; + +public interface ModelProviderRepository { + + /** insert model provider */ + Long insert(ModelProvider record); + + /** delete by id */ + int deleteById(Long id); + + /** update model provider */ + Long update(ModelProvider record); + + /** get by id */ + ModelProvider getById(Long id); + + /** get by provider */ + ModelProvider getByProvider(String provider); + + /** get by provider list */ + List selectByProviders(List providerList); + + /** query by condition */ + List query(ModelProviderQuery record); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/providerparam/ProviderParamRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/providerparam/ProviderParamRepository.java new file mode 100644 index 000000000..77c7696ee --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/providerparam/ProviderParamRepository.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.providerparam; + +import com.antgroup.openspg.server.common.model.providerparam.ProviderParam; +import com.antgroup.openspg.server.common.model.providerparam.ProviderParamQuery; +import java.util.List; + +public interface ProviderParamRepository { + + /** insert provider param */ + Long insert(ProviderParam record); + + /** delete by id */ + int deleteById(Long id); + + /** update provider param */ + Long update(ProviderParam record); + + /** get by id */ + ProviderParam getById(Long id); + + /** query provider param */ + List query(ProviderParamQuery record); + + /** get by provider and model type */ + ProviderParam getByProviderAndModelType(String provider, String modelType); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/ref/RefRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/ref/RefRepository.java new file mode 100644 index 000000000..327f33ee3 --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/ref/RefRepository.java @@ -0,0 +1,58 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.common.service.ref; + +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.common.model.ref.RefTypeEnum; +import com.antgroup.openspg.server.common.model.ref.RefedTypeEnum; +import java.util.List; + +public interface RefRepository { + + /** insert reference info */ + Long insert(RefInfo refInfo); + + /** delete by primary key */ + int deleteById(Long id); + + /** delete by multiple primary keys */ + int deleteByIds(List ids); + + /** update refInfo by primary key */ + int update(RefInfo refInfo); + + /** get refInfo by primary key */ + RefInfo getById(Long id); + + /** select by unique key */ + RefInfo selectByUniqueKey(String refId, String refType, String refedId, String refedType); + + /** update by primary key selective */ + int updateByPrimaryKeySelective(RefInfo refInfo); + + /** update by unique key */ + int updateByUniqueKey(RefInfo refInfo); + + /** get refInfo */ + List getRefInfoByRef(String refId, RefTypeEnum refType); + + /** get refInfo */ + List getRefInfoByRefed(String refedId, RefedTypeEnum refedType); + + /** Get a list of RefInfo objects by right matching (suffix) the refedId. */ + List getRefInfoByRightMatchRefedId(String refedId); + + /** Delete a RefInfo object by unique key. */ + int deleteByUniqueKey(RefInfo refInfo); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/RetrievalRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/RetrievalRepository.java new file mode 100644 index 000000000..e2d374fd8 --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/RetrievalRepository.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.retrieval; + +import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.common.model.retrieval.RetrievalQuery; + +public interface RetrievalRepository { + + /** insert Retrieval */ + Long insert(Retrieval record); + + /** delete By Id */ + int deleteById(Long id); + + /** update Job */ + Long update(Retrieval record); + + /** get By id */ + Retrieval getById(Long id); + + /** get By name */ + Retrieval getByName(String name); + + /** query By Condition */ + Paged query(RetrievalQuery record); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/RetrievalService.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/RetrievalService.java new file mode 100644 index 000000000..998d6703c --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/RetrievalService.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.retrieval; + +import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.common.model.retrieval.RetrievalQuery; +import java.util.List; + +public interface RetrievalService { + + /** insert Retrieval */ + Long insert(Retrieval record); + + /** delete By Id */ + int deleteById(Long id); + + /** update Retrieval */ + Long update(Retrieval record); + + /** get By id */ + Retrieval getById(Long id); + + /** get By name */ + Retrieval getByName(String name); + + /** query By Condition */ + Paged query(RetrievalQuery record); + + /** get project retrieval */ + List getRetrievalByProjectId(Long projectId); +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/impl/RetrievalServiceImpl.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/impl/RetrievalServiceImpl.java new file mode 100644 index 000000000..c93f99b30 --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/retrieval/impl/RetrievalServiceImpl.java @@ -0,0 +1,95 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.retrieval.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.server.common.model.bulider.BuilderJob; +import com.antgroup.openspg.server.common.model.bulider.BuilderJobQuery; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.common.model.retrieval.RetrievalQuery; +import com.antgroup.openspg.server.common.service.builder.BuilderJobService; +import com.antgroup.openspg.server.common.service.retrieval.RetrievalRepository; +import com.antgroup.openspg.server.common.service.retrieval.RetrievalService; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class RetrievalServiceImpl implements RetrievalService { + + @Autowired private RetrievalRepository retrievalRepository; + + @Autowired private BuilderJobService builderJobService; + + @Override + public Long insert(Retrieval record) { + return retrievalRepository.insert(record); + } + + @Override + public int deleteById(Long id) { + return retrievalRepository.deleteById(id); + } + + @Override + public Long update(Retrieval record) { + return retrievalRepository.update(record); + } + + @Override + public Retrieval getById(Long id) { + return retrievalRepository.getById(id); + } + + @Override + public Retrieval getByName(String name) { + return retrievalRepository.getByName(name); + } + + @Override + public Paged query(RetrievalQuery record) { + return retrievalRepository.query(record); + } + + @Override + public List getRetrievalByProjectId(Long projectId) { + BuilderJobQuery query = new BuilderJobQuery(); + query.setProjectId(projectId); + List jobs = builderJobService.query(query).getResults(); + Set ids = Sets.newHashSet(); + jobs.forEach( + job -> { + if (StringUtils.isNotBlank(job.getRetrievals())) { + List retrievals = + JSON.parseObject(job.getRetrievals(), new TypeReference>() {}); + ids.addAll(retrievals); + } + }); + if (CollectionUtils.isEmpty(ids)) { + return Lists.newArrayList(); + } + RetrievalQuery record = new RetrievalQuery(); + record.setIds(ids.stream().collect(Collectors.toList())); + return this.query(record).getResults(); + } +} diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/usermodel/UserModelRepository.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/usermodel/UserModelRepository.java new file mode 100644 index 000000000..87088e734 --- /dev/null +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/usermodel/UserModelRepository.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.common.service.usermodel; + +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.server.common.model.usermodel.UserModel; +import com.antgroup.openspg.server.common.model.usermodel.UserModelQuery; +import java.util.List; + +public interface UserModelRepository { + + /** insert user model */ + Long insert(UserModel record); + + /** delete By Id */ + int deleteById(Long id); + + /** delete By Ids */ + int deleteByIds(List ids); + + /** update user model */ + Long update(UserModel record); + + /** get By id */ + UserModel getById(Long id); + + /** + * get by instance id + * + * @param instanceId + * @return + */ + UserModel getByInstanceId(String instanceId); + + /** query By Condition */ + List query(UserModelQuery record); + + /** get By Provider And Name */ + List getByProviderAndName(String provider, String name); + + /** + * select user private or public + * + * @param userNo + * @return + */ + List selectUserPrivateOrPublic(String userNo); + + /** + * update base info by ids + * + * @param ids + * @param name + * @param visibility + * @param userNo + * @param config + * @return + */ + Long updateBaseInfoByIds( + List ids, String name, String visibility, String userNo, String config); + + /** + * get by model unique id + * + * @param modelId + * @return + */ + JSONObject getByModelId(String modelId); +} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/Utils.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/Utils.java index 948080141..f1b1b27e1 100644 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/Utils.java +++ b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/Utils.java @@ -77,10 +77,13 @@ public static List getAllRdfEntity( if (CollectionUtils.isNotEmpty(edgeList)) { for (IEdge edge : edgeList) { Object toIdObj = edge.getValue().get(Constants.EDGE_TO_ID_KEY); + String toIdType = (String) edge.getValue().get(Constants.EDGE_TO_ID_TYPE_KEY); + String dir = Direction.OUT.name(); Object nodeIdObj = vertex.getValue().get(Constants.NODE_ID_KEY); + String nodeType = vertex.getId().getType(); String targetType = edge.getTargetId().getType(); - if (nodeIdObj.equals(toIdObj)) { + if (nodeIdObj.equals(toIdObj) && nodeType.equals(toIdType)) { toIdObj = edge.getValue().get(Constants.EDGE_FROM_ID_KEY); dir = Direction.IN.name(); targetType = String.valueOf(edge.getValue().get(Constants.EDGE_FROM_ID_TYPE_KEY)); diff --git a/server/core/scheduler/model/src/main/java/com/antgroup/openspg/server/core/scheduler/model/task/TaskExecuteDag.java b/server/core/scheduler/model/src/main/java/com/antgroup/openspg/server/core/scheduler/model/task/TaskExecuteDag.java index 503aa840b..563acad86 100644 --- a/server/core/scheduler/model/src/main/java/com/antgroup/openspg/server/core/scheduler/model/task/TaskExecuteDag.java +++ b/server/core/scheduler/model/src/main/java/com/antgroup/openspg/server/core/scheduler/model/task/TaskExecuteDag.java @@ -122,7 +122,7 @@ public static class Node { private String taskComponent; /** properties */ - private JSONObject properties; + private JSONObject properties = new JSONObject(); } @Getter diff --git a/server/core/scheduler/service/pom.xml b/server/core/scheduler/service/pom.xml index aa717406a..42a31c85a 100644 --- a/server/core/scheduler/service/pom.xml +++ b/server/core/scheduler/service/pom.xml @@ -63,5 +63,9 @@ com.antgroup.openspg.builder builder-runner-local + + com.antgroup.openspg.server + biz-schema + diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/common/SchedulerCommonService.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/common/SchedulerCommonService.java index e2e7fa2c3..ae0bb251f 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/common/SchedulerCommonService.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/common/SchedulerCommonService.java @@ -12,9 +12,19 @@ */ package com.antgroup.openspg.server.core.scheduler.service.common; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.cloudext.interfaces.objectstorage.ObjectStorageClient; +import com.antgroup.openspg.cloudext.interfaces.objectstorage.ObjectStorageClientDriverManager; +import com.antgroup.openspg.common.constants.BuilderConstant; +import com.antgroup.openspg.common.util.CommonUtils; +import com.antgroup.openspg.common.util.pemja.PemjaUtils; +import com.antgroup.openspg.common.util.pemja.PythonInvokeMethod; +import com.antgroup.openspg.common.util.pemja.model.PemjaConfig; import com.antgroup.openspg.server.common.model.exception.SchedulerException; import com.antgroup.openspg.server.common.model.scheduler.SchedulerEnum.InstanceStatus; import com.antgroup.openspg.server.common.model.scheduler.SchedulerEnum.TaskStatus; +import com.antgroup.openspg.server.common.service.config.DefaultValue; import com.antgroup.openspg.server.common.service.spring.SpringContextHolder; import com.antgroup.openspg.server.core.scheduler.model.query.SchedulerInstanceQuery; import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerInstance; @@ -31,6 +41,7 @@ import com.antgroup.openspg.server.core.scheduler.service.translate.TranslatorFactory; import com.antgroup.openspg.server.core.scheduler.service.utils.SchedulerUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.util.Date; import java.util.List; import java.util.Map; @@ -51,6 +62,8 @@ public class SchedulerCommonService { public static final String UNDERLINE_SEPARATOR = "_"; public static final Long FINISH = 100L; + @Autowired DefaultValue value; + @Autowired SchedulerJobService schedulerJobService; @Autowired SchedulerInstanceService schedulerInstanceService; @@ -61,6 +74,7 @@ public class SchedulerCommonService { public void setInstanceFinish( SchedulerInstance instance, InstanceStatus instanceStatus, TaskStatus taskStatus) { Date finishTime = (instance.getFinishTime() == null ? new Date() : instance.getFinishTime()); + costStatistics(instance); SchedulerInstance updateInstance = new SchedulerInstance(); updateInstance.setId(instance.getId()); updateInstance.setStatus(instanceStatus); @@ -220,4 +234,61 @@ public SchedulerInstance generateInstance(SchedulerJob job, String uniqueId, Dat return instance; } + + public void costStatistics(SchedulerInstance instance) { + SchedulerInstance old = schedulerInstanceService.getById(instance.getId()); + JSONObject extension = old.getExtension(); + extension = (extension == null) ? new JSONObject() : extension; + PemjaConfig pemjaConfig = + new PemjaConfig( + value.getPythonExec(), + value.getPythonPaths(), + value.getPythonEnv(), + value.getSchemaUrlHost(), + null, + PythonInvokeMethod.BRIDGE_GET_LLM_TOKEN_INFO, + Maps.newHashMap()); + Object result = PemjaUtils.invoke(pemjaConfig, instance.getId()); + JSONObject tokens = JSON.parseObject(JSON.toJSONString(result)); + if (tokens != null) { + Integer token = tokens.getInteger(BuilderConstant.COMPLETION_TOKENS); + extension.put(BuilderConstant.TOKENS_COST, token.toString()); + } + + Date finishTime = (instance.getFinishTime() == null ? new Date() : instance.getFinishTime()); + long diffInMillis = finishTime.getTime() - instance.getGmtCreate().getTime(); + long hours = diffInMillis / (1000 * 60 * 60); + long minutes = (diffInMillis % (1000 * 60 * 60)) / (1000 * 60); + long seconds = (diffInMillis % (1000 * 60)) / 1000; + String time = String.format("%sh %sm %ss", hours, minutes, seconds); + extension.put(BuilderConstant.TIME_COST, time); + + ObjectStorageClient objectStorageClient = + ObjectStorageClientDriverManager.getClient(value.getObjectStorageUrl()); + long totalSizeInBytes = + objectStorageClient.getStorageSize( + value.getBuilderBucketName(), + CommonUtils.getInstanceStorageFileKey(instance.getProjectId(), instance.getId())); + extension.put(BuilderConstant.STORAGE_COST, formatBytes(totalSizeInBytes)); + + SchedulerInstance updateInstance = new SchedulerInstance(); + updateInstance.setId(instance.getId()); + updateInstance.setExtension(extension); + schedulerInstanceService.update(updateInstance); + } + + public static String formatBytes(long sizeInBytes) { + final String[] units = {"Bytes", "KB", "MB", "GB", "TB", "PB"}; + final double base = 1024.0; + if (sizeInBytes < base) { + return sizeInBytes + " Bytes"; + } + int unitIndex = 0; + double size = sizeInBytes; + while (size >= base && unitIndex < units.length - 1) { + size /= base; + unitIndex++; + } + return String.format("%.2f %s", size, units[unitIndex]); + } } diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/ComputingEngineAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/ComputingEngineAsyncTask.java index ea572fd5c..f25110e53 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/ComputingEngineAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/ComputingEngineAsyncTask.java @@ -90,18 +90,26 @@ public static void putOrDefault(JSONObject extension, String key, Object default public JSONObject initExtension(BuilderJob builderJob, SchedulerInstance instance) { JSONObject extension = JSONObject.parseObject(builderJob.getComputingConf()); String command = extension.getString(BuilderConstant.COMMAND); + String url = + StringUtils.isBlank(value.getWebRequestUrl()) + ? value.getSchemaUrlHost() + : value.getWebRequestUrl(); if (StringUtils.isBlank(command) && !BuilderConstant.KAG_COMMAND.equals(builderJob.getType())) { Project project = projectManager.queryById(builderJob.getProjectId()); - JSONObject config = - CommonUtils.getKagBuilderConfig(project, builderJob, value.getSchemaUrlHost()); + JSONObject config = CommonUtils.getKagBuilderConfig(project, builderJob, url); String input = CommonUtils.getKagBuilderInput(builderJob, instance.getSchedulerDate()); extension.put( BuilderConstant.COMMAND, - String.format(BuilderConstant.SPG_DEFAULT_COMMAND, config.toJSONString(), input)); + String.format( + BuilderConstant.SPG_DEFAULT_COMMAND, + builderJob.getProjectId(), + url, + config.toJSONString(), + input)); } putOrDefault(extension, BuilderConstants.PYTHON_EXEC_OPTION, value.getPythonExec()); putOrDefault(extension, BuilderConstants.PYTHON_PATHS_OPTION, value.getPythonPaths()); - putOrDefault(extension, BuilderConstants.SCHEMA_URL_OPTION, value.getSchemaUrlHost()); + putOrDefault(extension, BuilderConstants.SCHEMA_URL_OPTION, url); putOrDefault(extension, BuilderConstants.PARALLELISM_OPTION, 1); String jobAction = (builderJob.getAction() != null) @@ -242,10 +250,8 @@ public SchedulerEnum.TaskStatus processByFinished(TaskExecuteContext context, St public Boolean stop(TaskExecuteContext context, String resource) { ComputingEngineClient client = ComputingEngineClientDriverManager.getClient(value.getComputingEngineUrl()); - SchedulerJob job = context.getJob(); - BuilderJob builderJob = builderJobService.getById(Long.valueOf(job.getInvokerId())); JSONObject extension = new JSONObject(); - extension.put(ComputingEngineConstants.USER_NUMBER, builderJob.getModifyUser()); + extension.put(ComputingEngineConstants.USER_NUMBER, ComputingEngineConstants.DEFAULT_NUMBER); return client.stop(extension, resource); } } diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagAlignmentAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagAlignmentAsyncTask.java index a8f8726e9..7bf57da22 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagAlignmentAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagAlignmentAsyncTask.java @@ -24,6 +24,7 @@ import com.antgroup.openspg.common.util.pemja.PemjaUtils; import com.antgroup.openspg.common.util.pemja.PythonInvokeMethod; import com.antgroup.openspg.common.util.pemja.model.PemjaConfig; +import com.antgroup.openspg.server.common.model.CommonConstants; import com.antgroup.openspg.server.common.model.scheduler.SchedulerEnum; import com.antgroup.openspg.server.common.service.config.DefaultValue; import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerInstance; @@ -36,9 +37,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.PostConstruct; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -46,18 +55,43 @@ @Component("kagAlignmentAsyncTask") public class KagAlignmentAsyncTask extends AsyncTaskExecuteTemplate { + private static final RejectedExecutionHandler handler = + (r, executor) -> { + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + + private static ThreadPoolExecutor executor; + @Autowired private DefaultValue value; @Autowired private MemoryTaskServer memoryTaskServer; @Autowired private SchedulerTaskService taskService; + @PostConstruct + public void init() { + if (executor == null) { + executor = + new ThreadPoolExecutor( + value.getModelExecuteNum(), + value.getModelExecuteNum(), + 60 * 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), + handler); + } + } + @Override public String submit(TaskExecuteContext context) { SchedulerInstance instance = context.getInstance(); SchedulerTask task = context.getTask(); String key = - CommonUtils.getTaskStorageFileKey( + CommonUtils.getTaskStoragePathKey( task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); SchedulerTask memoryTask = memoryTaskServer.getTask(key); if (memoryTask != null) { @@ -92,6 +126,8 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res switch (task.getStatus()) { case RUNNING: break; + case WAIT: + return SchedulerEnum.TaskStatus.RUNNING; case ERROR: int retryNum = 3; if (schedulerTask.getExecuteNum() % retryNum == 0) { @@ -151,38 +187,96 @@ public VectorizerTaskCallable( @Override public String call() throws Exception { - List subGraphList = Lists.newArrayList(); addTraceLog("Start Alignment task!"); - for (String input : inputs) { - String data = objectStorageClient.getString(value.getBuilderBucketName(), input); - List subGraphs = - JSON.parseObject(data, new TypeReference>() {}); - subGraphList.addAll(alignment(context, subGraphs)); - } + SchedulerTask task = context.getTask(); + Long projectId = context.getInstance().getProjectId(); + String pathKey = + CommonUtils.getTaskStoragePathKey( + task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); + + JSONObject pyConfig = getPyConfig(); + PemjaConfig pemjaConfig = getPemjaConfig(projectId); AtomicLong nodes = new AtomicLong(0); AtomicLong edges = new AtomicLong(0); - SchedulerUtils.getGraphSize(subGraphList, nodes, edges); + List> futures = new ArrayList<>(); + + for (Integer i = 0; i < inputs.size(); i++) { + final Integer inputIndex = i; + List files = + objectStorageClient.getAllFilesRecursively(value.getBuilderBucketName(), inputs.get(i)); + + for (Integer f = 0; f < files.size(); f++) { + final Integer fileIndex = f; + final String filePath = files.get(f); + addTraceLog( + "Alignment ThreadPool. size:%s active:%s completed:%s total:%s queue:%s", + executor.getPoolSize(), + executor.getActiveCount(), + executor.getCompletedTaskCount(), + executor.getTaskCount(), + executor.getQueue().size()); + Future future = + executor.submit( + () -> { + String data = + objectStorageClient.getString(value.getBuilderBucketName(), filePath); + SubGraphRecord subGraph = JSON.parseObject(data, SubGraphRecord.class); + addTraceLog( + "Invoke the alignment operator. index:%s/%s", fileIndex + 1, files.size()); + subGraph = alignment(pyConfig, pemjaConfig, subGraph); + addTraceLog( + "Alignment operator was invoked successfully index:%s/%s nodes:%s edges:%s", + fileIndex + 1, + files.size(), + subGraph.getResultNodes().size(), + subGraph.getResultEdges().size()); + SchedulerUtils.getGraphSize(subGraph, nodes, edges); + String fileKey = + CommonUtils.getTaskStorageFileKey(pathKey, inputIndex + "_" + fileIndex); + objectStorageClient.saveString( + value.getBuilderBucketName(), JSON.toJSONString(subGraph), fileKey); + addTraceLog( + "Store the results of the alignment operator. file:%s/%s", + value.getBuilderBucketName(), fileKey); + }); + futures.add(future); + } + } + for (Future future : futures) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("invoke alignment Exception", e); + } + } addTraceLog("Alignment task complete. nodes:%s. edges:%s", nodes.get(), edges.get()); - SchedulerTask task = context.getTask(); - String fileKey = - CommonUtils.getTaskStorageFileKey( - task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); - objectStorageClient.saveString( - value.getBuilderBucketName(), JSON.toJSONString(subGraphList), fileKey); - addTraceLog( - "Store the results of the alignment operator. file:%s/%s", - value.getBuilderBucketName(), fileKey); - return fileKey; + return pathKey; } - public List alignment( - TaskExecuteContext context, List subGraphs) { - List subGraphList = Lists.newArrayList(); - Long projectId = context.getInstance().getProjectId(); - PythonInvokeMethod alignment = PythonInvokeMethod.BRIDGE_COMPONENT; - JSONObject pyConfig = new JSONObject(); - pyConfig.put(BuilderConstant.TYPE, BuilderConstant.BASE); + public SubGraphRecord alignment( + JSONObject pyConfig, PemjaConfig pemjaConfig, SubGraphRecord subGraph) { + SubGraphRecord record = new SubGraphRecord(Lists.newArrayList(), Lists.newArrayList()); + Map map = new ObjectMapper().convertValue(subGraph, Map.class); + List result = + (List) + PemjaUtils.invoke( + pemjaConfig, BuilderConstant.POSTPROCESSOR_ABC, pyConfig.toJSONString(), map); + List records = + JSON.parseObject(JSON.toJSONString(result), new TypeReference>() {}); + + for (SubGraphRecord subGraphRecord : records) { + if (CollectionUtils.isNotEmpty(subGraphRecord.getResultNodes())) { + record.getResultNodes().addAll(subGraphRecord.getResultNodes()); + } + if (CollectionUtils.isNotEmpty(subGraphRecord.getResultEdges())) { + record.getResultEdges().addAll(subGraphRecord.getResultEdges()); + } + } + return record; + } + private PemjaConfig getPemjaConfig(Long projectId) { + PythonInvokeMethod alignment = PythonInvokeMethod.BRIDGE_COMPONENT; PemjaConfig pemjaConfig = new PemjaConfig( value.getPythonExec(), @@ -192,33 +286,14 @@ public List alignment( projectId, alignment, Maps.newHashMap()); - int index = 0; - for (SubGraphRecord subGraph : subGraphs) { - addTraceLog("Invoke the alignment operator. index:%s/%s", ++index, subGraphs.size()); - Map map = new ObjectMapper().convertValue(subGraph, Map.class); - List result = - (List) - PemjaUtils.invoke( - pemjaConfig, BuilderConstant.POSTPROCESSOR_ABC, pyConfig.toJSONString(), map); - List records = - JSON.parseObject( - JSON.toJSONString(result), new TypeReference>() {}); - subGraphList.addAll(records); - for (SubGraphRecord subGraphRecord : records) { - int nodes = - CollectionUtils.isEmpty(subGraphRecord.getResultNodes()) - ? 0 - : subGraphRecord.getResultNodes().size(); - int edges = - CollectionUtils.isEmpty(subGraphRecord.getResultEdges()) - ? 0 - : subGraphRecord.getResultEdges().size(); + return pemjaConfig; + } - addTraceLog( - "Alignment operator was invoked successfully nodes:%s edges:%s", nodes, edges); - } - } - return subGraphList; + private JSONObject getPyConfig() { + JSONObject pyConfig = new JSONObject(); + pyConfig.put(BuilderConstant.TYPE, BuilderConstant.BASE); + pyConfig.put(CommonConstants.TASK_ID, context.getInstance().getId()); + return pyConfig; } } } diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagBuilderAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagBuilderAsyncTask.java index f78601fc5..b521b0d2f 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagBuilderAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagBuilderAsyncTask.java @@ -83,6 +83,8 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res switch (task.getStatus()) { case RUNNING: break; + case WAIT: + return SchedulerEnum.TaskStatus.RUNNING; case ERROR: int retryNum = 3; if (schedulerTask.getExecuteNum() % retryNum == 0) { diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagExtractorAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagExtractorAsyncTask.java index fe726ff67..987ed3090 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagExtractorAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagExtractorAsyncTask.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.server.core.scheduler.service.task.async.builder; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; import com.antgroup.openspg.builder.model.record.ChunkRecord; @@ -36,6 +37,7 @@ import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerJob; import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerTask; import com.antgroup.openspg.server.core.scheduler.model.task.TaskExecuteContext; +import com.antgroup.openspg.server.core.scheduler.model.task.TaskExecuteDag; import com.antgroup.openspg.server.core.scheduler.service.common.MemoryTaskServer; import com.antgroup.openspg.server.core.scheduler.service.metadata.SchedulerTaskService; import com.antgroup.openspg.server.core.scheduler.service.task.async.AsyncTaskExecuteTemplate; @@ -65,6 +67,8 @@ public class KagExtractorAsyncTask extends AsyncTaskExecuteTemplate { public static final String LLM_ID = "llm_id"; + public static final String CONFIG = "config"; + public static final String EXTRACTOR = "extractor"; private static final RejectedExecutionHandler handler = (r, executor) -> { @@ -93,7 +97,7 @@ private static ThreadPoolExecutor createExecutor(DefaultValue value) { value.getModelExecuteNum(), 60, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(100), + new LinkedBlockingQueue<>(1000), handler); } @@ -102,7 +106,7 @@ public String submit(TaskExecuteContext context) { SchedulerInstance instance = context.getInstance(); SchedulerTask task = context.getTask(); String key = - CommonUtils.getTaskStorageFileKey( + CommonUtils.getTaskStoragePathKey( task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); SchedulerTask memoryTask = memoryTaskServer.getTask(key); if (memoryTask != null) { @@ -143,6 +147,8 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res switch (task.getStatus()) { case RUNNING: break; + case WAIT: + return SchedulerEnum.TaskStatus.RUNNING; case ERROR: int retryNum = 3; if (schedulerTask.getExecuteNum() % retryNum == 0) { @@ -154,7 +160,7 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res break; case FINISH: String fileKey = - CommonUtils.getTaskStorageFileKey( + CommonUtils.getTaskStoragePathKey( schedulerTask.getProjectId(), schedulerTask.getInstanceId(), schedulerTask.getId(), @@ -219,8 +225,10 @@ public String call() throws Exception { JSON.parseObject(data, new TypeReference>() {}); chunkList.addAll(chunks); } - addTraceLog("Start extract document. chunk size:%s", chunkList.size()); - + TaskExecuteDag.Node node = + context.getInstance().getTaskDag().getNode(context.getTask().getNodeId()); + String name = node.getProperties().getString("retrievalName"); + addTraceLog("Start extract(%s) document. chunk size:%s", name, chunkList.size()); List>> futures = new ArrayList<>(); List results = new ArrayList<>(); int index = 0; @@ -235,15 +243,22 @@ public String call() throws Exception { executor.getCompletedTaskCount(), executor.getTaskCount(), executor.getQueue().size()); - Future> future = - executor.submit(new ExtractTaskCallable(chunk, value, llm, project, indexStr)); - futures.add(future); + JSONArray extractors = getPyConfig(project, value, llm, name); + for (int i = 0; i < extractors.size(); i++) { + JSONObject pyConfig = extractors.getJSONObject(i); + pyConfig.put(CommonConstants.TASK_ID, context.getInstance().getId()); + Future> future = + executor.submit(new ExtractTaskCallable(chunk, value, pyConfig, project, indexStr)); + futures.add(future); + } } for (Future> future : futures) { try { List result = future.get(); - results.addAll(result); + result.forEach( + subGraphRecord -> + SchedulerUtils.addSubGraph(results, subGraphRecord, value.getSubGraphBatchMax())); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException("invoke extract Exception", e); } @@ -254,33 +269,62 @@ public String call() throws Exception { addTraceLog("Extract document complete. nodes:%s. edges:%s", nodes.get(), edges.get()); SchedulerTask task = context.getTask(); - String fileKey = - CommonUtils.getTaskStorageFileKey( + String pathKey = + CommonUtils.getTaskStoragePathKey( task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); - objectStorageClient.saveString( - value.getBuilderBucketName(), JSON.toJSONString(results), fileKey); - addTraceLog( - "Store the results of the extract operator. file:%s/%s", - value.getBuilderBucketName(), fileKey); - return fileKey; + for (Integer i = 0; i < results.size(); i++) { + String fileKey = CommonUtils.getTaskStorageFileKey(pathKey, i.toString()); + objectStorageClient.saveString( + value.getBuilderBucketName(), JSON.toJSONString(results.get(i)), fileKey); + addTraceLog( + "Store the results of the extract operator. file:%s/%s", + value.getBuilderBucketName(), fileKey); + } + return pathKey; + } + + public JSONArray getPyConfig(Project project, DefaultValue value, JSONObject llm, String name) { + if (StringUtils.isBlank(name) || "_default".equals(name)) { + JSONArray extractors = new JSONArray(); + JSONObject pyConfig = new JSONObject(); + pyConfig.put(BuilderConstant.TYPE, BuilderConstant.SCHEMA_FREE); + pyConfig.put(BuilderConstant.LLM, this.llm); + extractors.add(pyConfig); + return extractors; + } + PemjaConfig pemjaConfig = + new PemjaConfig( + value.getPythonExec(), + value.getPythonPaths(), + value.getPythonEnv(), + value.getSchemaUrlHost(), + project.getId(), + PythonInvokeMethod.BRIDGE_GET_INDEX_MANAGER_INFO, + Maps.newHashMap()); + JSONObject vec = + JSONObject.parseObject(project.getConfig()).getJSONObject(CommonConstants.VECTORIZER); + Object result = PemjaUtils.invoke(pemjaConfig, name, llm.toJSONString(), vec.toJSONString()); + JSONObject info = JSON.parseObject(result.toString()); + JSONArray extractors = info.getJSONObject(CONFIG).getJSONArray(EXTRACTOR); + return extractors; } class ExtractTaskCallable implements Callable> { private final ChunkRecord.Chunk chunk; private final DefaultValue value; - private final JSONObject llm; + private final JSONObject pyConfig; private final Project project; private final String indexStr; public ExtractTaskCallable( ChunkRecord.Chunk chunk, DefaultValue value, - JSONObject llm, + JSONObject pyConfig, Project project, String indexStr) { this.chunk = chunk; this.value = value; - this.llm = llm; + this.pyConfig = pyConfig; this.project = project; this.indexStr = indexStr; } @@ -288,9 +332,6 @@ public ExtractTaskCallable( @Override public List call() throws Exception { PythonInvokeMethod extractor = PythonInvokeMethod.BRIDGE_COMPONENT; - JSONObject pyConfig = new JSONObject(); - pyConfig.put(BuilderConstant.TYPE, BuilderConstant.SCHEMA_FREE); - pyConfig.put(BuilderConstant.LLM, this.llm); PemjaConfig pemjaConfig = new PemjaConfig( value.getPythonExec(), @@ -308,7 +349,10 @@ public List call() throws Exception { List result = (List) PemjaUtils.invoke( - pemjaConfig, BuilderConstant.EXTRACTOR_ABC, pyConfig.toJSONString(), map); + pemjaConfig, + BuilderConstant.EXTRACTOR_ABC, + this.pyConfig.toJSONString(), + map); records = JSON.parseObject( JSON.toJSONString(result), new TypeReference>() {}); @@ -342,7 +386,7 @@ public List call() throws Exception { ? 0 : subGraphRecord.getResultEdges().size(); addTraceLog( - "Extract chunk(%s:%s) index:%s successfully. nodes:%s. edges:%s", + "Extract chunk(%s:%s) successfully. index:%s nodes:%s. edges:%s", chunk.getName(), chunk.getShortId(), indexStr, nodes, edges); } return records; diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagSplitterAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagSplitterAsyncTask.java index 1a7211d22..9be86acf9 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagSplitterAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagSplitterAsyncTask.java @@ -23,6 +23,7 @@ import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.common.util.pemja.PemjaUtils; import com.antgroup.openspg.common.util.pemja.model.PemjaConfig; +import com.antgroup.openspg.server.common.model.CommonConstants; import com.antgroup.openspg.server.common.model.bulider.BuilderJob; import com.antgroup.openspg.server.common.model.project.Project; import com.antgroup.openspg.server.common.model.scheduler.SchedulerEnum; @@ -98,6 +99,8 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res switch (task.getStatus()) { case RUNNING: break; + case WAIT: + return SchedulerEnum.TaskStatus.RUNNING; case ERROR: int retryNum = 3; if (schedulerTask.getExecuteNum() % retryNum == 0) { @@ -186,6 +189,7 @@ public List splitterChunk( TaskExecuteContext context, List chunks) { List chunkList = Lists.newArrayList(); JSONObject pyConfig = new JSONObject(); + pyConfig.put(CommonConstants.TASK_ID, context.getInstance().getId()); Project project = projectService.queryById(context.getInstance().getProjectId()); SchedulerJob job = context.getJob(); BuilderJob builderJob = builderJobService.getById(Long.valueOf(job.getInvokerId())); diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagVectorizerAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagVectorizerAsyncTask.java index ade5477e4..dd596acd7 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagVectorizerAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagVectorizerAsyncTask.java @@ -39,9 +39,17 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.PostConstruct; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -49,6 +57,17 @@ @Component("kagVectorizerAsyncTask") public class KagVectorizerAsyncTask extends AsyncTaskExecuteTemplate { + private static final RejectedExecutionHandler handler = + (r, executor) -> { + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + + private static ThreadPoolExecutor executor; + @Autowired private DefaultValue value; @Autowired private MemoryTaskServer memoryTaskServer; @@ -57,12 +76,26 @@ public class KagVectorizerAsyncTask extends AsyncTaskExecuteTemplate { @Autowired private ProjectService projectService; + @PostConstruct + public void init() { + if (executor == null) { + executor = + new ThreadPoolExecutor( + value.getModelExecuteNum(), + value.getModelExecuteNum(), + 60 * 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), + handler); + } + } + @Override public String submit(TaskExecuteContext context) { SchedulerInstance instance = context.getInstance(); SchedulerTask task = context.getTask(); String key = - CommonUtils.getTaskStorageFileKey( + CommonUtils.getTaskStoragePathKey( task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); SchedulerTask memoryTask = memoryTaskServer.getTask(key); if (memoryTask != null) { @@ -99,6 +132,8 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res switch (task.getStatus()) { case RUNNING: break; + case WAIT: + return SchedulerEnum.TaskStatus.RUNNING; case ERROR: int retryNum = 3; if (schedulerTask.getExecuteNum() % retryNum == 0) { @@ -152,45 +187,102 @@ public VectorizerTaskCallable( @Override public String call() throws Exception { - List subGraphList = Lists.newArrayList(); addTraceLog("Start vectorized document!"); - for (String input : inputs) { - String data = objectStorageClient.getString(value.getBuilderBucketName(), input); - List subGraphs = - JSON.parseObject(data, new TypeReference>() {}); - subGraphList.addAll(vectorizer(context, subGraphs)); - } + SchedulerTask task = context.getTask(); + Long projectId = context.getInstance().getProjectId(); + String pathKey = + CommonUtils.getTaskStoragePathKey( + task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); + + JSONObject pyConfig = getPyConfig(projectId); + PemjaConfig pemjaConfig = getPemjaConfig(projectId); AtomicLong nodes = new AtomicLong(0); AtomicLong edges = new AtomicLong(0); - SchedulerUtils.getGraphSize(subGraphList, nodes, edges); + + List> futures = new ArrayList<>(); + + for (Integer i = 0; i < inputs.size(); i++) { + final Integer inputIndex = i; + List files = + objectStorageClient.getAllFilesRecursively(value.getBuilderBucketName(), inputs.get(i)); + + for (Integer f = 0; f < files.size(); f++) { + final Integer fileIndex = f; + final String filePath = files.get(f); + addTraceLog( + "Vector ThreadPool. size:%s active:%s completed:%s total:%s queue:%s", + executor.getPoolSize(), + executor.getActiveCount(), + executor.getCompletedTaskCount(), + executor.getTaskCount(), + executor.getQueue().size()); + Future future = + executor.submit( + () -> { + String data = + objectStorageClient.getString(value.getBuilderBucketName(), filePath); + SubGraphRecord subGraph = JSON.parseObject(data, SubGraphRecord.class); + addTraceLog( + "Invoke the vector operator. index:%s/%s", fileIndex + 1, files.size()); + subGraph = vectorizer(pyConfig, pemjaConfig, subGraph); + addTraceLog( + "Vector operator was invoked successfully index:%s/%s nodes:%s edges:%s", + fileIndex + 1, + files.size(), + subGraph.getResultNodes().size(), + subGraph.getResultEdges().size()); + SchedulerUtils.getGraphSize(subGraph, nodes, edges); + String fileKey = + CommonUtils.getTaskStorageFileKey(pathKey, inputIndex + "_" + fileIndex); + String results = JSON.toJSONString(subGraph); + Long start = System.currentTimeMillis(); + byte[] bytes = results.getBytes(StandardCharsets.UTF_8); + objectStorageClient.saveData(value.getBuilderBucketName(), bytes, fileKey); + addTraceLog( + "Store the results of the vector operator. file:%s/%s length:%s cons:%s", + value.getBuilderBucketName(), + fileKey, + bytes.length, + System.currentTimeMillis() - start); + }); + futures.add(future); + } + } + for (Future future : futures) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("invoke vector Exception", e); + } + } + addTraceLog("Vectorized document complete. nodes:%s. edges:%s", nodes.get(), edges.get()); - SchedulerTask task = context.getTask(); - String fileKey = - CommonUtils.getTaskStorageFileKey( - task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); - String results = JSON.toJSONString(subGraphList); - Long statr = System.currentTimeMillis(); - byte[] bytes = results.getBytes(StandardCharsets.UTF_8); - addTraceLog("Start Store the results of the vector operator! byte length:%s", bytes.length); - objectStorageClient.saveData(value.getBuilderBucketName(), bytes, fileKey); - addTraceLog( - "Store the results of the vector operator. file:%s/%s cons:%s", - value.getBuilderBucketName(), fileKey, System.currentTimeMillis() - statr); - return fileKey; + return pathKey; } - public List vectorizer( - TaskExecuteContext context, List subGraphs) { - List subGraphList = Lists.newArrayList(); - Long projectId = context.getInstance().getProjectId(); - String projectConfig = projectService.queryById(projectId).getConfig(); - JSONObject vec = - JSONObject.parseObject(projectConfig).getJSONObject(CommonConstants.VECTORIZER); - PythonInvokeMethod vectorizer = PythonInvokeMethod.BRIDGE_COMPONENT; - JSONObject pyConfig = new JSONObject(); - pyConfig.put(BuilderConstant.TYPE, BuilderConstant.BATCH); - pyConfig.put(BuilderConstant.VECTORIZE_MODEL, vec); + public SubGraphRecord vectorizer( + JSONObject pyConfig, PemjaConfig pemjaConfig, SubGraphRecord subGraph) { + SubGraphRecord record = new SubGraphRecord(Lists.newArrayList(), Lists.newArrayList()); + Map map = new ObjectMapper().convertValue(subGraph, Map.class); + List result = + (List) + PemjaUtils.invoke( + pemjaConfig, BuilderConstant.VECTORIZER_ABC, pyConfig.toJSONString(), map); + List records = + JSON.parseObject(JSON.toJSONString(result), new TypeReference>() {}); + for (SubGraphRecord subGraphRecord : records) { + if (CollectionUtils.isNotEmpty(subGraphRecord.getResultNodes())) { + record.getResultNodes().addAll(subGraphRecord.getResultNodes()); + } + if (CollectionUtils.isNotEmpty(subGraphRecord.getResultEdges())) { + record.getResultEdges().addAll(subGraphRecord.getResultEdges()); + } + } + return record; + } + private PemjaConfig getPemjaConfig(Long projectId) { + PythonInvokeMethod vectorizer = PythonInvokeMethod.BRIDGE_COMPONENT; PemjaConfig pemjaConfig = new PemjaConfig( value.getPythonExec(), @@ -200,31 +292,18 @@ public List vectorizer( projectId, vectorizer, Maps.newHashMap()); - int index = 0; - for (SubGraphRecord subGraph : subGraphs) { - addTraceLog("Invoke the vector operator. index:%s/%s", ++index, subGraphs.size()); - Map map = new ObjectMapper().convertValue(subGraph, Map.class); - List result = - (List) - PemjaUtils.invoke( - pemjaConfig, BuilderConstant.VECTORIZER_ABC, pyConfig.toJSONString(), map); - List records = - JSON.parseObject( - JSON.toJSONString(result), new TypeReference>() {}); - subGraphList.addAll(records); - for (SubGraphRecord subGraphRecord : records) { - int nodes = - CollectionUtils.isEmpty(subGraphRecord.getResultNodes()) - ? 0 - : subGraphRecord.getResultNodes().size(); - int edges = - CollectionUtils.isEmpty(subGraphRecord.getResultEdges()) - ? 0 - : subGraphRecord.getResultEdges().size(); - addTraceLog("Vector operator was invoked successfully nodes:%s edges:%s", nodes, edges); - } - } - return subGraphList; + return pemjaConfig; + } + + private JSONObject getPyConfig(Long projectId) { + String projectConfig = projectService.queryById(projectId).getConfig(); + JSONObject vec = + JSONObject.parseObject(projectConfig).getJSONObject(CommonConstants.VECTORIZER); + JSONObject pyConfig = new JSONObject(); + pyConfig.put(BuilderConstant.TYPE, BuilderConstant.BATCH); + pyConfig.put(BuilderConstant.VECTORIZE_MODEL, vec); + pyConfig.put(CommonConstants.TASK_ID, context.getInstance().getId()); + return pyConfig; } } } diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagWriterAsyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagWriterAsyncTask.java index 2b9c90de0..6a48a3713 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagWriterAsyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/async/builder/KagWriterAsyncTask.java @@ -13,7 +13,6 @@ package com.antgroup.openspg.server.core.scheduler.service.task.async.builder; import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.TypeReference; import com.antgroup.openspg.builder.core.runtime.BuilderContext; import com.antgroup.openspg.builder.model.pipeline.config.Neo4jSinkNodeConfig; import com.antgroup.openspg.builder.model.record.RecordAlterOperationEnum; @@ -36,13 +35,20 @@ import com.antgroup.openspg.server.core.scheduler.service.metadata.SchedulerTaskService; import com.antgroup.openspg.server.core.scheduler.service.task.async.AsyncTaskExecuteTemplate; import com.antgroup.openspg.server.core.scheduler.service.utils.SchedulerUtils; -import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.PostConstruct; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -50,6 +56,17 @@ @Component("kagWriterAsyncTask") public class KagWriterAsyncTask extends AsyncTaskExecuteTemplate { + private static final RejectedExecutionHandler handler = + (r, executor) -> { + try { + executor.getQueue().put(r); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + + private static ThreadPoolExecutor executor; + @Autowired private DefaultValue value; @Autowired private ProjectService projectManager; @@ -60,12 +77,26 @@ public class KagWriterAsyncTask extends AsyncTaskExecuteTemplate { @Autowired private BuilderJobService builderJobService; + @PostConstruct + public void init() { + if (executor == null) { + executor = + new ThreadPoolExecutor( + value.getModelExecuteNum(), + value.getModelExecuteNum(), + 60 * 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), + handler); + } + } + @Override public String submit(TaskExecuteContext context) { SchedulerInstance instance = context.getInstance(); SchedulerTask task = context.getTask(); String key = - CommonUtils.getTaskStorageFileKey( + CommonUtils.getTaskStoragePathKey( task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); SchedulerTask memoryTask = memoryTaskServer.getTask(key); if (memoryTask != null) { @@ -103,6 +134,8 @@ public SchedulerEnum.TaskStatus getStatus(TaskExecuteContext context, String res switch (task.getStatus()) { case RUNNING: break; + case WAIT: + return SchedulerEnum.TaskStatus.RUNNING; case ERROR: int retryNum = 3; if (schedulerTask.getExecuteNum() % retryNum == 0) { @@ -175,91 +208,122 @@ public WriterTaskCallable( @Override public String call() throws Exception { - List subGraphList = Lists.newArrayList(); addTraceLog("Start write task!"); + SchedulerTask task = context.getTask(); + String pathKey = + CommonUtils.getTaskStoragePathKey( + task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); + + AtomicLong nodes = new AtomicLong(0); + AtomicLong edges = new AtomicLong(0); + + List> futures = new ArrayList<>(); + RecordAlterOperationEnum action = RecordAlterOperationEnum.UPSERT; if (RecordAlterOperationEnum.DELETE.name().equalsIgnoreCase(this.action)) { action = RecordAlterOperationEnum.DELETE; } - for (String input : inputs) { - String data = objectStorageClient.getString(value.getBuilderBucketName(), input); - List subGraphs = - JSON.parseObject(data, new TypeReference>() {}); - writer(value, projectManager, context, action, subGraphs); - subGraphList.addAll(simpleSubGraph(subGraphs)); + Neo4jSinkWriter writer = getNeo4jSinkWriter(value, projectManager, context, action); + for (Integer i = 0; i < inputs.size(); i++) { + final Integer inputIndex = i; + List files = + objectStorageClient.getAllFilesRecursively(value.getBuilderBucketName(), inputs.get(i)); + + for (Integer f = 0; f < files.size(); f++) { + final Integer fileIndex = f; + final String filePath = files.get(f); + addTraceLog( + "Write ThreadPool. size:%s active:%s completed:%s total:%s queue:%s", + executor.getPoolSize(), + executor.getActiveCount(), + executor.getCompletedTaskCount(), + executor.getTaskCount(), + executor.getQueue().size()); + Future future = + executor.submit( + () -> { + String data = + objectStorageClient.getString(value.getBuilderBucketName(), filePath); + SubGraphRecord subGraph = JSON.parseObject(data, SubGraphRecord.class); + String indexStr = (fileIndex + 1) + "/" + files.size(); + addTraceLog("Invoke the write operator. index:%s", indexStr); + writer(writer, subGraph, indexStr); + simpleSubGraph(subGraph); + SchedulerUtils.getGraphSize(subGraph, nodes, edges); + String fileKey = + CommonUtils.getTaskStorageFileKey(pathKey, inputIndex + "_" + fileIndex); + objectStorageClient.saveString( + value.getBuilderBucketName(), JSON.toJSONString(subGraph), fileKey); + addTraceLog( + "Store the results of the alignment operator. file:%s/%s", + value.getBuilderBucketName(), fileKey); + }); + futures.add(future); + } } - AtomicLong nodes = new AtomicLong(0); - AtomicLong edges = new AtomicLong(0); - SchedulerUtils.getGraphSize(subGraphList, nodes, edges); + for (Future future : futures) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("invoke write Exception", e); + } + } + addTraceLog("Write task complete. nodes:%s. edges:%s", nodes.get(), edges.get()); - SchedulerTask task = context.getTask(); - String fileKey = - CommonUtils.getTaskStorageFileKey( - task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); - objectStorageClient.saveString( - value.getBuilderBucketName(), JSON.toJSONString(subGraphList), fileKey); - addTraceLog( - "Store the results of the write operator. file:%s/%s", - value.getBuilderBucketName(), fileKey); - return fileKey; + return pathKey; } - public List simpleSubGraph(List subGraphs) { - for (SubGraphRecord subGraph : subGraphs) { - subGraph - .getResultNodes() - .forEach( - node -> { - Iterator> iterator = - node.getProperties().entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (entry.getKey().endsWith(VECTOR)) { - iterator.remove(); - } + public void simpleSubGraph(SubGraphRecord subGraph) { + subGraph + .getResultNodes() + .forEach( + node -> { + Iterator> iterator = + node.getProperties().entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getKey().endsWith(VECTOR)) { + iterator.remove(); } - }); - } - return subGraphs; + } + }); + + return; } - public void writer( - DefaultValue value, - ProjectService projectManager, - TaskExecuteContext context, - RecordAlterOperationEnum action, - List subGraphs) { - Neo4jSinkWriter writer = - new Neo4jSinkWriter( - UUID.randomUUID().toString(), "writer", new Neo4jSinkNodeConfig(true)); - BuilderContext builderContext = - new BuilderContext() - .setProjectId(context.getInstance().getProjectId()) - .setJobName("writer") - .setPythonExec(value.getPythonExec()) - .setPythonPaths(value.getPythonPaths()) - .setPythonEnv(value.getPythonEnv()) - .setOperation(action) - .setEnableLeadTo(false) - .setProject( - JSON.toJSONString(projectManager.queryById(context.getInstance().getProjectId()))) - .setGraphStoreUrl( - projectManager.getGraphStoreUrl(context.getInstance().getProjectId())); - writer.init(builderContext); - int index = 0; - for (SubGraphRecord subGraph : subGraphs) { - addTraceLog("Invoke the write operator. index:%s/%s", ++index, subGraphs.size()); - writer.writeToNeo4j(subGraph); - int nodes = - CollectionUtils.isEmpty(subGraph.getResultNodes()) - ? 0 - : subGraph.getResultNodes().size(); - int edges = - CollectionUtils.isEmpty(subGraph.getResultEdges()) - ? 0 - : subGraph.getResultEdges().size(); - addTraceLog("Write operator was invoked successfully nodes:%s edges:%s", nodes, edges); - } + public void writer(Neo4jSinkWriter writer, SubGraphRecord subGraph, String indexStr) { + writer.writeToNeo4j(subGraph); + int nodes = + CollectionUtils.isEmpty(subGraph.getResultNodes()) ? 0 : subGraph.getResultNodes().size(); + int edges = + CollectionUtils.isEmpty(subGraph.getResultEdges()) ? 0 : subGraph.getResultEdges().size(); + addTraceLog( + "Write operator was invoked successfully index:%s nodes:%s edges:%s", + indexStr, nodes, edges); } } + + private static Neo4jSinkWriter getNeo4jSinkWriter( + DefaultValue value, + ProjectService projectManager, + TaskExecuteContext context, + RecordAlterOperationEnum action) { + Neo4jSinkWriter writer = + new Neo4jSinkWriter(UUID.randomUUID().toString(), "writer", new Neo4jSinkNodeConfig(true)); + BuilderContext builderContext = + new BuilderContext() + .setProjectId(context.getInstance().getProjectId()) + .setJobName("writer") + .setPythonExec(value.getPythonExec()) + .setPythonPaths(value.getPythonPaths()) + .setPythonEnv(value.getPythonEnv()) + .setOperation(action) + .setEnableLeadTo(false) + .setProject( + JSON.toJSONString(projectManager.queryById(context.getInstance().getProjectId()))) + .setGraphStoreUrl( + projectManager.getGraphStoreUrl(context.getInstance().getProjectId())); + writer.init(builderContext); + return writer; + } } diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/KagMappingSyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/KagMappingSyncTask.java index 0c6cd4f80..cb5e7023c 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/KagMappingSyncTask.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/KagMappingSyncTask.java @@ -61,36 +61,43 @@ public SchedulerEnum.TaskStatus submit(TaskExecuteContext context) { objectStorageClient = ObjectStorageClientDriverManager.getClient(value.getObjectStorageUrl()); SchedulerJob job = context.getJob(); BuilderJob builderJob = builderJobService.getById(Long.valueOf(job.getInvokerId())); - List chunks = loadData(context, builderJob); + List results = loadData(context, builderJob); + SchedulerTask task = context.getTask(); - String fileKey = - com.antgroup.openspg.common.util.CommonUtils.getTaskStorageFileKey( + String pathKey = + CommonUtils.getTaskStoragePathKey( task.getProjectId(), task.getInstanceId(), task.getId(), task.getType()); - objectStorageClient.saveString( - value.getBuilderBucketName(), JSON.toJSONString(chunks), fileKey); - context.addTraceLog( - "Store the results of the mapping operator. file:%s/%s", - value.getBuilderBucketName(), fileKey); - task.setOutput(fileKey); + for (Integer i = 0; i < results.size(); i++) { + String fileKey = CommonUtils.getTaskStorageFileKey(pathKey, i.toString()); + objectStorageClient.saveString( + value.getBuilderBucketName(), JSON.toJSONString(results.get(i)), fileKey); + context.addTraceLog( + "Store the results of the mapping operator. file:%s/%s", + value.getBuilderBucketName(), fileKey); + } + task.setOutput(pathKey); return SchedulerEnum.TaskStatus.FINISH; } public List loadData(TaskExecuteContext context, BuilderJob builderJob) { SchedulerInstance instance = context.getInstance(); SchedulerTask task = context.getTask(); + List results = Lists.newArrayList(); List inputs = SchedulerUtils.getTaskInputs(taskService, instance, task); - List subGraphList = Lists.newArrayList(); for (String input : inputs) { String data = objectStorageClient.getString(value.getBuilderBucketName(), input); List> datas = JSON.parseObject(data, new TypeReference>>() {}); - subGraphList.addAll(mapping(context, datas, builderJob)); + List result = mapping(context, datas, builderJob); + result.forEach( + subGraphRecord -> + SchedulerUtils.addSubGraph(results, subGraphRecord, value.getSubGraphBatchMax())); } AtomicLong nodes = new AtomicLong(0); AtomicLong edges = new AtomicLong(0); - SchedulerUtils.getGraphSize(subGraphList, nodes, edges); + SchedulerUtils.getGraphSize(results, nodes, edges); context.addTraceLog("Data mapping complete. nodes:%s. edges:%s", nodes.get(), edges.get()); - return subGraphList; + return results; } public List mapping( diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/RetrievalSyncTask.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/RetrievalSyncTask.java new file mode 100644 index 000000000..41279dc68 --- /dev/null +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/task/sync/builder/RetrievalSyncTask.java @@ -0,0 +1,135 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.core.scheduler.service.task.sync.builder; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.core.schema.model.alter.SchemaDraft; +import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; +import com.antgroup.openspg.core.schema.model.type.BaseSPGType; +import com.antgroup.openspg.core.schema.model.type.ProjectSchema; +import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; +import com.antgroup.openspg.server.api.facade.dto.schema.request.SchemaAlterRequest; +import com.antgroup.openspg.server.biz.schema.SchemaManager; +import com.antgroup.openspg.server.biz.schema.model.NodeTypeModel; +import com.antgroup.openspg.server.biz.schema.model.SchemaCompareUtil; +import com.antgroup.openspg.server.biz.schema.model.SchemaModel; +import com.antgroup.openspg.server.biz.schema.model.SchemaModelConvertor; +import com.antgroup.openspg.server.biz.schema.model.SchemaScriptTranslateUtil; +import com.antgroup.openspg.server.common.model.bulider.BuilderJob; +import com.antgroup.openspg.server.common.model.project.Project; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.common.model.scheduler.SchedulerEnum; +import com.antgroup.openspg.server.common.service.builder.BuilderJobService; +import com.antgroup.openspg.server.common.service.project.ProjectService; +import com.antgroup.openspg.server.common.service.retrieval.RetrievalService; +import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerJob; +import com.antgroup.openspg.server.core.scheduler.model.task.TaskExecuteContext; +import com.antgroup.openspg.server.core.scheduler.service.task.sync.SyncTaskExecuteTemplate; +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component("retrievalSyncTask") +public class RetrievalSyncTask extends SyncTaskExecuteTemplate { + + private static final List PAGE_DISPLAY_TYPE = + Lists.newArrayList( + SPGTypeEnum.ENTITY_TYPE, + SPGTypeEnum.EVENT_TYPE, + SPGTypeEnum.CONCEPT_TYPE, + SPGTypeEnum.INDEX_TYPE); + + @Autowired private BuilderJobService builderJobService; + + @Autowired private RetrievalService retrievalService; + + @Autowired private ProjectService projectService; + + @Autowired private SchemaManager schemaManager; + + @Override + public SchedulerEnum.TaskStatus submit(TaskExecuteContext context) { + + SchedulerJob job = context.getJob(); + BuilderJob builderJob = builderJobService.getById(Long.valueOf(job.getInvokerId())); + String retrievals = builderJob.getRetrievals(); + if (StringUtils.isBlank(retrievals)) { + context.addTraceLog("index not set"); + return SchedulerEnum.TaskStatus.FINISH; + } + context.addTraceLog("update index schema index_ids:%s", retrievals); + List retrievalList = JSON.parseObject(retrievals, new TypeReference>() {}); + Project project = projectService.queryById(job.getProjectId()); + for (Long id : retrievalList) { + Retrieval retrieval = retrievalService.getById(id); + context.addTraceLog("update index(%s) schema", retrieval.getName()); + String schemaDesc = retrieval.getSchemaDesc(); + if (StringUtils.isBlank(schemaDesc)) { + continue; + } + SchemaDraft schemaDraft = getSpgSchema(project, schemaDesc); + SchemaAlterRequest request = new SchemaAlterRequest(); + request.setProjectId(project.getId()); + request.setSchemaDraft(schemaDraft); + schemaManager.alterSchema(request); + context.addTraceLog("update index(%s) schema succeed", retrieval.getName()); + } + return SchedulerEnum.TaskStatus.FINISH; + } + + private SchemaDraft getSpgSchema(Project project, String schema) { + SchemaModel schemaModel = + SchemaScriptTranslateUtil.translateScript( + "namespace " + project.getNamespace() + "\n\n" + schema); + + ProjectSchema projectSchema = schemaManager.getProjectSchema(project.getId()); + List oldNodeTypeModels = Lists.newArrayList(); + for (BaseSPGType spgType : projectSchema.getSpgTypes()) { + if (PAGE_DISPLAY_TYPE.contains(spgType.getSpgTypeEnum())) { + NodeTypeModel entityModel = + SchemaModelConvertor.convert2NodeTypeModel( + project.getNamespace(), (BaseAdvancedType) spgType); + oldNodeTypeModels.add(entityModel); + } + } + List nodeTypeModels = schemaModel.getNodeTypeModels(); + Set nameSet = + nodeTypeModels.stream().map(NodeTypeModel::getName).collect(Collectors.toSet()); + for (NodeTypeModel model : oldNodeTypeModels) { + if (!nameSet.contains(model.getName())) { + nodeTypeModels.add(model); + } + } + SchemaCompareUtil schemaCompareUtil = new SchemaCompareUtil(); + SchemaCompareUtil.SchemaChangeDTO changeDTO = + schemaCompareUtil.compare(schemaModel.getNamespace(), oldNodeTypeModels, nodeTypeModels); + log.info( + "RetrievalSyncTask changeDTO-ADD({})-DEL({})-UPDATE({})", + changeDTO.getAddTypes().size(), + changeDTO.getDeleteTypes().size(), + changeDTO.getUpdateTypes().size()); + SchemaDraft schemaDraft = new SchemaDraft(); + List newSchema = Lists.newArrayList(); + newSchema.addAll(changeDTO.getAddTypes()); + newSchema.addAll(changeDTO.getUpdateTypes()); + schemaDraft.setAlterSpgTypes(newSchema); + return schemaDraft; + } +} diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/translate/builder/KagBuilderTranslate.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/translate/builder/KagBuilderTranslate.java index 11a4c6ac6..c4e0494cc 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/translate/builder/KagBuilderTranslate.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/translate/builder/KagBuilderTranslate.java @@ -12,13 +12,18 @@ */ package com.antgroup.openspg.server.core.scheduler.service.translate.builder; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.TypeReference; import com.antgroup.openspg.cloudext.interfaces.objectstorage.ObjectStorageClient; import com.antgroup.openspg.cloudext.interfaces.objectstorage.ObjectStorageClientDriverManager; import com.antgroup.openspg.common.constants.BuilderConstant; import com.antgroup.openspg.server.common.model.bulider.BuilderJob; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; import com.antgroup.openspg.server.common.model.scheduler.SchedulerEnum; import com.antgroup.openspg.server.common.service.builder.BuilderJobService; import com.antgroup.openspg.server.common.service.config.DefaultValue; +import com.antgroup.openspg.server.common.service.retrieval.RetrievalService; import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerInstance; import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerJob; import com.antgroup.openspg.server.core.scheduler.model.service.SchedulerTask; @@ -28,6 +33,7 @@ import com.google.common.collect.Lists; import java.util.List; import java.util.UUID; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -41,6 +47,8 @@ public class KagBuilderTranslate implements Translate { @Autowired private DefaultValue value; + @Autowired private RetrievalService retrievalService; + @Override public TaskExecuteDag translate(SchedulerJob schedulerJob) { BuilderJob builderJob = builderJobService.getById(Long.valueOf(schedulerJob.getInvokerId())); @@ -75,9 +83,21 @@ public TaskExecuteDag getTaskDag(BuilderJob builderJob) { List nodes = Lists.newArrayList(); List edges = Lists.newArrayList(); + String retrievals = builderJob.getRetrievals(); + List retrievalList = Lists.newArrayList(); + if (StringUtils.isNotBlank(retrievals)) { + retrievalList = JSON.parseObject(retrievals, new TypeReference>() {}); + } TaskExecuteDag taskDag = new TaskExecuteDag(); + TaskExecuteDag.Node retrievalNode = new TaskExecuteDag.Node(); + String retrievalId = UUID.randomUUID().toString(); + retrievalNode.setId(retrievalId); + retrievalNode.setName("Create Index"); + retrievalNode.setTaskComponent("retrievalSyncTask"); + nodes.add(retrievalNode); + String checkPartitionId = UUID.randomUUID().toString(); if (BuilderConstant.ODPS.equalsIgnoreCase(builderJob.getDataSourceType())) { TaskExecuteDag.Node checkPartition = new TaskExecuteDag.Node(); @@ -101,12 +121,34 @@ public TaskExecuteDag getTaskDag(BuilderJob builderJob) { splitter.setTaskComponent("kagSplitterAsyncTask"); nodes.add(splitter); - TaskExecuteDag.Node extractor = new TaskExecuteDag.Node(); - String extractorId = UUID.randomUUID().toString(); - extractor.setId(extractorId); - extractor.setName("Extractor"); - extractor.setTaskComponent("kagExtractorAsyncTask"); - nodes.add(extractor); + List extractorIds = Lists.newArrayList(); + if (CollectionUtils.isEmpty(retrievalList)) { + TaskExecuteDag.Node extractor = new TaskExecuteDag.Node(); + String extractorId = UUID.randomUUID().toString(); + extractorIds.add(extractorId); + extractor.setId(extractorId); + extractor.setName("Extractor"); + extractor.setTaskComponent("kagExtractorAsyncTask"); + JSONObject properties = new JSONObject(); + properties.put("retrievalName", "_default"); + extractor.setProperties(properties); + nodes.add(extractor); + } else { + for (Long id : retrievalList) { + Retrieval retrieval = retrievalService.getById(id); + TaskExecuteDag.Node extractor = new TaskExecuteDag.Node(); + String extractorId = UUID.randomUUID().toString(); + extractorIds.add(extractorId); + extractor.setId(extractorId); + extractor.setName("Extractor(" + retrieval.getName() + ")"); + extractor.setTaskComponent("kagExtractorAsyncTask_" + retrieval.getName()); + JSONObject properties = new JSONObject(); + properties.put("retrievalId", retrieval.getId()); + properties.put("retrievalName", retrieval.getName()); + extractor.setProperties(properties); + nodes.add(extractor); + } + } TaskExecuteDag.Node vectorizer = new TaskExecuteDag.Node(); String vectorizerId = UUID.randomUUID().toString(); @@ -129,11 +171,18 @@ public TaskExecuteDag getTaskDag(BuilderJob builderJob) { writer.setTaskComponent("kagWriterAsyncTask"); nodes.add(writer); + TaskExecuteDag.Edge retrievalEdge = new TaskExecuteDag.Edge(); + retrievalEdge.setFrom(retrievalId); if (BuilderConstant.ODPS.equalsIgnoreCase(builderJob.getDataSourceType())) { + retrievalEdge.setTo(checkPartitionId); + edges.add(retrievalEdge); TaskExecuteDag.Edge checkPartitionEdge = new TaskExecuteDag.Edge(); checkPartitionEdge.setFrom(checkPartitionId); checkPartitionEdge.setTo(readerId); edges.add(checkPartitionEdge); + } else { + retrievalEdge.setTo(readerId); + edges.add(retrievalEdge); } TaskExecuteDag.Edge edge = new TaskExecuteDag.Edge(); @@ -141,15 +190,17 @@ public TaskExecuteDag getTaskDag(BuilderJob builderJob) { edge.setTo(splitterId); edges.add(edge); - TaskExecuteDag.Edge edge1 = new TaskExecuteDag.Edge(); - edge1.setFrom(splitterId); - edge1.setTo(extractorId); - edges.add(edge1); + for (String extractorId : extractorIds) { + TaskExecuteDag.Edge edge1 = new TaskExecuteDag.Edge(); + edge1.setFrom(splitterId); + edge1.setTo(extractorId); + edges.add(edge1); - TaskExecuteDag.Edge edge2 = new TaskExecuteDag.Edge(); - edge2.setFrom(extractorId); - edge2.setTo(vectorizerId); - edges.add(edge2); + TaskExecuteDag.Edge edge2 = new TaskExecuteDag.Edge(); + edge2.setFrom(extractorId); + edge2.setTo(vectorizerId); + edges.add(edge2); + } TaskExecuteDag.Edge edge3 = new TaskExecuteDag.Edge(); edge3.setFrom(vectorizerId); diff --git a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/utils/SchedulerUtils.java b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/utils/SchedulerUtils.java index 5965b4778..cf67730ee 100644 --- a/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/utils/SchedulerUtils.java +++ b/server/core/scheduler/service/src/main/java/com/antgroup/openspg/server/core/scheduler/service/utils/SchedulerUtils.java @@ -166,6 +166,20 @@ public static void getGraphSize( } } + public static void getGraphSize(SubGraphRecord record, AtomicLong nodes, AtomicLong edges) { + nodes = (nodes == null) ? new AtomicLong(0) : nodes; + edges = (edges == null) ? new AtomicLong(0) : edges; + + List resultNodes = record.getResultNodes(); + if (CollectionUtils.isNotEmpty(resultNodes)) { + nodes.addAndGet(resultNodes.size()); + } + List resultEdges = record.getResultEdges(); + if (CollectionUtils.isNotEmpty(resultEdges)) { + edges.addAndGet(resultEdges.size()); + } + } + public static List getTaskInputs( SchedulerTaskService taskService, SchedulerInstance instance, SchedulerTask task) { List nodes = @@ -181,4 +195,35 @@ public static List getTaskInputs( }); return inputs; } + + public static void addSubGraph( + List result, SubGraphRecord newRecord, Integer subGraphBatchMax) { + if (CollectionUtils.isEmpty(result)) { + result.add(newRecord); + return; + } + if (CollectionUtils.isEmpty(newRecord.getResultNodes())) { + newRecord.setResultNodes(Lists.newArrayList()); + } + if (CollectionUtils.isEmpty(newRecord.getResultEdges())) { + newRecord.setResultEdges(Lists.newArrayList()); + } + + SubGraphRecord lastRecord = result.get(result.size() - 1); + if (CollectionUtils.isEmpty(lastRecord.getResultNodes())) { + lastRecord.setResultNodes(Lists.newArrayList()); + } + if (CollectionUtils.isEmpty(lastRecord.getResultEdges())) { + lastRecord.setResultEdges(Lists.newArrayList()); + } + + int totalNodesAndEdges = lastRecord.getResultNodes().size(); + + if (totalNodesAndEdges + newRecord.getResultNodes().size() > subGraphBatchMax) { + result.add(newRecord); + } else { + lastRecord.getResultNodes().addAll(newRecord.getResultNodes()); + lastRecord.getResultEdges().addAll(newRecord.getResultEdges()); + } + } } diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/identifier/ConceptIdentifier.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/identifier/ConceptIdentifier.java index 2c8bf69c0..1d6c235f7 100644 --- a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/identifier/ConceptIdentifier.java +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/identifier/ConceptIdentifier.java @@ -24,7 +24,7 @@ public class ConceptIdentifier extends BaseSPGIdentifier { /** The seperator symbol between parent and child concept names. */ public static final String SEPARATOR = "-"; - public static final String ROOT = "__ROOT__"; + public static final String ROOT = "_root"; /** Unique id of concept node. */ private final String id; diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/IndexTypeEnum.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/IndexTypeEnum.java index 2dee3793a..98fa40330 100644 --- a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/IndexTypeEnum.java +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/IndexTypeEnum.java @@ -19,11 +19,17 @@ public enum IndexTypeEnum { /** vector index. */ VECTOR("VECTOR", "Vector"), + /** sparse vector index. */ + SPARSE_VECTOR("SPARSE_VECTOR", "SparseVector"), + /** text index. */ TEXT("TEXT", "Text"), /** text and vector index. */ - TEXT_AND_VECTOR("TEXT_AND_VECTOR", "TextAndVector"); + TEXT_AND_VECTOR("TEXT_AND_VECTOR", "TextAndVector"), + + /** text and sparse vector index. */ + TEXT_AND_SPARSE_VECTOR("TEXT_AND_SPARSE_VECTOR", "TextAndSparseVector"); /** Name of index. */ private final String nameEn; diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseAdvancedType.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseAdvancedType.java index 0011dc9c4..d99e52afd 100644 --- a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseAdvancedType.java +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseAdvancedType.java @@ -36,6 +36,7 @@ *
  • entity type: {@link EntityType} *
  • concept type: {@link ConceptType} *
  • event type: {@link EventType} + *
  • index type: {@link IndexType} * */ public abstract class BaseAdvancedType extends BaseSPGType { diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseSPGType.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseSPGType.java index 2f6451429..a5ff1856c 100644 --- a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseSPGType.java +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/BaseSPGType.java @@ -42,6 +42,7 @@ *
  • {@link EntityType}：such as User,Company etc. *
  • {@link ConceptType}：such as brand,Administration etc. *
  • {@link EventType}：such as Epidemic event,Terrorist incident etc. + *
  • {@link IndexType}: such as summary index, chunk2query index etc * * *
    diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/IndexType.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/IndexType.java new file mode 100644 index 000000000..bb13b337f --- /dev/null +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/IndexType.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.core.schema.model.type; + +import com.antgroup.openspg.core.schema.model.BasicInfo; +import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; +import com.antgroup.openspg.core.schema.model.predicate.Property; +import com.antgroup.openspg.core.schema.model.predicate.Relation; +import java.util.List; + +/** + * Class definition of the Index.
    + * + *

    this is a index type + */ +public class IndexType extends BaseAdvancedType { + + private static final long serialVersionUID = 7548382967010252528L; + + public IndexType( + BasicInfo basicInfo, + ParentTypeInfo parentTypeInfo, + List properties, + List relations, + SPGTypeAdvancedConfig advancedConfig) { + super(basicInfo, parentTypeInfo, SPGTypeEnum.INDEX_TYPE, properties, relations, advancedConfig); + } +} diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/SPGTypeEnum.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/SPGTypeEnum.java index 656b54d13..bd9c10fc4 100644 --- a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/SPGTypeEnum.java +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/type/SPGTypeEnum.java @@ -22,6 +22,9 @@ public enum SPGTypeEnum { /** @see EntityType */ ENTITY_TYPE, + /** @see IndexType */ + INDEX_TYPE, + /** @see ConceptType */ CONCEPT_TYPE, diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/IndexTypeChecker.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/IndexTypeChecker.java new file mode 100644 index 000000000..d79ec92ac --- /dev/null +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/IndexTypeChecker.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.core.schema.service.alter.check; + +import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; +import com.antgroup.openspg.core.schema.model.type.IndexType; + +public class IndexTypeChecker extends BaseSpgTypeChecker { + @Override + public void checkAdvancedConfig(BaseAdvancedType advancedType, SchemaCheckContext context) { + IndexType indexType = (IndexType) advancedType; + String schemaTypeName = indexType.getName(); + OperatorChecker.check(schemaTypeName, indexType.getAdvancedConfig().getLinkOperator()); + OperatorChecker.check(schemaTypeName, indexType.getAdvancedConfig().getFuseOperator()); + } +} diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/SchemaAlterChecker.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/SchemaAlterChecker.java index fcebbccfa..901e52304 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/SchemaAlterChecker.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/SchemaAlterChecker.java @@ -31,6 +31,7 @@ public SchemaAlterChecker() { validatorMap.put(SPGTypeEnum.ENTITY_TYPE, new EntityTypeChecker()); validatorMap.put(SPGTypeEnum.CONCEPT_TYPE, new ConceptTypeChecker()); validatorMap.put(SPGTypeEnum.EVENT_TYPE, new EventTypeChecker()); + validatorMap.put(SPGTypeEnum.INDEX_TYPE, new IndexTypeChecker()); } public void check(SchemaCheckContext context) { diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PostProcessStage.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PostProcessStage.java index 16992d251..b85c2edf9 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PostProcessStage.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PostProcessStage.java @@ -13,10 +13,12 @@ package com.antgroup.openspg.server.core.schema.service.alter.stage; +import com.antgroup.openspg.common.util.constants.CommonConstant; import com.antgroup.openspg.core.schema.model.SPGSchema; import com.antgroup.openspg.core.schema.model.SPGSchemaAlterCmd; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.type.BaseSPGType; +import com.antgroup.openspg.server.common.service.project.ProjectService; import com.antgroup.openspg.server.core.schema.service.alter.model.SchemaAlterContext; import com.antgroup.openspg.server.core.schema.service.alter.sync.BaseSchemaSyncer; import com.antgroup.openspg.server.core.schema.service.alter.sync.SchemaStorageEnum; @@ -28,6 +30,8 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; /** * Post process after saving schema metadata to db. such as notify other system、save snapshot etc. @@ -37,6 +41,7 @@ public class PostProcessStage extends BaseAlterStage { @Autowired private SchemaSyncerFactory schemaSyncerFactory; @Autowired private SPGTypeService spgTypeService; + @Autowired private ProjectService projectManager; /** Target storage to sync schema */ private static final List TARGET_SYNC_STORE = @@ -58,6 +63,12 @@ private void syncSchema(SchemaAlterContext context) { SPGSchemaAlterCmd schemaEditCmd = new SPGSchemaAlterCmd(new SPGSchema(spgTypes, spreadStdTypeNames)); + Long projectId = context.getProject().getId(); + String graphStoreEngineType = getGraphStoreEngineType(projectId); + // TODO: 这里先跳过，kgfabric driver schema/索引同步打通后放开 @秉初 底层支持索引灵活变更的接口5月底给出，排期530开发 + if (CommonConstant.KGFABRIC_GRAPH_STORE.equalsIgnoreCase(graphStoreEngineType)) { + return; + } for (SchemaStorageEnum targetStorage : TARGET_SYNC_STORE) { BaseSchemaSyncer schemaSyncer = schemaSyncerFactory.getSchemaSyncer(targetStorage); if (schemaSyncer != null) { @@ -65,4 +76,16 @@ private void syncSchema(SchemaAlterContext context) { } } } + + /** + * TODO：获取存储引擎类型的临时方法，对接kgfabric后下掉，这里不感知引擎类型 + * + * @param projectId + * @return + */ + private String getGraphStoreEngineType(Long projectId) { + String graphStoreUrl = projectManager.getGraphStoreUrl(projectId); + UriComponents uriComponents = UriComponentsBuilder.fromUriString(graphStoreUrl).build(); + return uriComponents.getScheme(); + } } diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PreProcessStage.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PreProcessStage.java index b7f33abff..648323807 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PreProcessStage.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/PreProcessStage.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.server.core.schema.service.alter.stage; +import com.antgroup.openspg.common.constants.SpgAppConstant; import com.antgroup.openspg.core.schema.model.SchemaConstants; import com.antgroup.openspg.core.schema.model.SchemaException; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; @@ -22,6 +23,7 @@ import com.antgroup.openspg.core.schema.model.type.BaseSPGType; import com.antgroup.openspg.core.schema.model.type.StandardType; import com.antgroup.openspg.core.schema.model.type.WithAlterOperation; +import com.antgroup.openspg.core.schema.model.type.WithBasicInfo; import com.antgroup.openspg.server.core.schema.service.alter.check.SchemaAlterChecker; import com.antgroup.openspg.server.core.schema.service.alter.check.SchemaCheckContext; import com.antgroup.openspg.server.core.schema.service.alter.check.SchemaMap; @@ -111,6 +113,12 @@ private void addInheritedProperty( BaseSPGType spgType, List parentTypes, Set spreadStandardTypeIdentifiers) { + List selfProp = + CollectionUtils.isEmpty(spgType.getProperties()) + ? new ArrayList<>() + : spgType.getProperties().stream() + .map(WithBasicInfo::getName) + .collect(Collectors.toList()); for (BaseSPGType parentType : parentTypes) { if (CollectionUtils.isEmpty(parentType.getProperties())) { continue; @@ -120,6 +128,10 @@ private void addInheritedProperty( if (Boolean.TRUE.equals(property.getInherited())) { continue; } + if (SpgAppConstant.DISPLAY_INHERIT_PROPERTIES.contains(property.getName()) + && selfProp.contains(property.getName())) { + continue; + } Property newProp = PropertyUtils.inheritProperty(spgType.toRef(), property); spgType.getProperties().add(newProp); diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeAssemble.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeAssemble.java index 94a1729df..04f94643f 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeAssemble.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeAssemble.java @@ -22,6 +22,7 @@ import com.antgroup.openspg.server.core.schema.service.util.PropertyUtils; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -83,6 +84,19 @@ private static List getPropertiesOfType( typeProperties.add(p); } } + typeProperties = + typeProperties.stream() + .collect( + Collectors.collectingAndThen( + Collectors.toMap( + Property::getName, + p -> p, + (existing, replacement) -> + existing.getInherited() && !replacement.getInherited() + ? replacement + : existing, + LinkedHashMap::new), + map -> new ArrayList<>(map.values()))); return typeProperties; } diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeConvertor.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeConvertor.java index cb7972a94..9d82d843b 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeConvertor.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/type/convertor/SPGTypeConvertor.java @@ -20,6 +20,7 @@ import com.antgroup.openspg.core.schema.model.type.ConceptType; import com.antgroup.openspg.core.schema.model.type.EntityType; import com.antgroup.openspg.core.schema.model.type.EventType; +import com.antgroup.openspg.core.schema.model.type.IndexType; import com.antgroup.openspg.core.schema.model.type.OperatorTypeEnum; import com.antgroup.openspg.core.schema.model.type.SPGTypeAdvancedConfig; import com.antgroup.openspg.core.schema.model.type.StandardType; @@ -183,6 +184,15 @@ public static BaseSPGType toBaseSpgType( relations, getAdvancedConfig(simpleSpgType)); break; + case INDEX_TYPE: + baseSpgType = + new IndexType( + simpleSpgType.getBasicInfo(), + simpleSpgType.getParentTypeInfo(), + propertys, + relations, + getAdvancedConfig(simpleSpgType)); + break; default: throw new IllegalArgumentException("illegal type=" + simpleSpgType.getSpgTypeIdentifier()); } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/AppDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/AppDO.java new file mode 100644 index 000000000..b7722ce10 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/AppDO.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class AppDO { + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String creator; + private String modifier; + private String name; + private String description; + private String logo; + private String config; + private String alias; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/BuilderJobDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/BuilderJobDO.java index 14bd008ca..845a65c67 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/BuilderJobDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/BuilderJobDO.java @@ -39,4 +39,5 @@ public class BuilderJobDO { private String computingConf; private String lifeCycle; private String action; + private String retrievals; } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/FeedbackDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/FeedbackDO.java new file mode 100644 index 000000000..21e1be9e0 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/FeedbackDO.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class FeedbackDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String moduleType; + private String oneCategory; + private String twoCategory; + private String threeCategory; + private String fourCategory; + private String fiveCategory; + private String reactionType; + private String reason; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ModelDetailDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ModelDetailDO.java new file mode 100644 index 000000000..32e005c87 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ModelDetailDO.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class ModelDetailDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String provider; + private String type; + private String name; + private String description; + private String params; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ModelProviderDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ModelProviderDO.java new file mode 100644 index 000000000..e8af8aca2 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ModelProviderDO.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class ModelProviderDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String name; + private String provider; + private String status; + private String pageMode; + private String modelType; + private String logo; + private String tags; + private String params; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProjectDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProjectDO.java index 7787c52b5..998a27615 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProjectDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProjectDO.java @@ -14,7 +14,9 @@ package com.antgroup.openspg.server.infra.dao.dataobject; import java.util.Date; +import lombok.Data; +@Data public class ProjectDO { private Long id; @@ -32,77 +34,9 @@ public class ProjectDO { private Long bizDomainId; - private String config; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description == null ? null : description.trim(); - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status == null ? null : status.trim(); - } + private String visibility; - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace == null ? null : namespace.trim(); - } - - public Long getBizDomainId() { - return bizDomainId; - } - - public void setBizDomainId(Long bizDomainId) { - this.bizDomainId = bizDomainId; - } - - public String getConfig() { - return config; - } + private String config; - public void setConfig(String config) { - this.config = config; - } + private String tag; } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProviderParamDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProviderParamDO.java new file mode 100644 index 000000000..a9a76b42d --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ProviderParamDO.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class ProviderParamDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String provider; + private String modelType; + private String modelName; + private String params; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/RefDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/RefDO.java new file mode 100644 index 000000000..d14b78b17 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/RefDO.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class RefDO { + private Long id; + + private Date gmtCreate; + + private Date gmtModified; + + private String name; + + private String refId; + + private String refType; + + private String refedId; + + private String refedType; + + private Integer status; + + private String config; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDO.java index 40676bd05..e5bc5173a 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDO.java @@ -26,7 +26,7 @@ public class ReleaseDO { private Integer version; - private String userId; + private String userNo; private String status; @@ -70,12 +70,12 @@ public void setVersion(Integer version) { this.version = version; } - public String getUserId() { - return userId; + public String getUserNo() { + return userNo; } - public void setUserId(String userId) { - this.userId = userId == null ? null : userId.trim(); + public void setUserNo(String userNo) { + this.userNo = userNo == null ? null : userNo.trim(); } public String getStatus() { diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDOExample.java index c85722b1a..fb0cdf4bc 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDOExample.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/ReleaseDOExample.java @@ -418,76 +418,6 @@ public Criteria andVersionNotBetween(Integer value1, Integer value2) { return (Criteria) this; } - public Criteria andUserIdIsNull() { - addCriterion("user_id is null"); - return (Criteria) this; - } - - public Criteria andUserIdIsNotNull() { - addCriterion("user_id is not null"); - return (Criteria) this; - } - - public Criteria andUserIdEqualTo(String value) { - addCriterion("user_id =", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdNotEqualTo(String value) { - addCriterion("user_id <>", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdGreaterThan(String value) { - addCriterion("user_id >", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdGreaterThanOrEqualTo(String value) { - addCriterion("user_id >=", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdLessThan(String value) { - addCriterion("user_id <", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdLessThanOrEqualTo(String value) { - addCriterion("user_id <=", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdLike(String value) { - addCriterion("user_id like", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdNotLike(String value) { - addCriterion("user_id not like", value, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdIn(List values) { - addCriterion("user_id in", values, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdNotIn(List values) { - addCriterion("user_id not in", values, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdBetween(String value1, String value2) { - addCriterion("user_id between", value1, value2, "userId"); - return (Criteria) this; - } - - public Criteria andUserIdNotBetween(String value1, String value2) { - addCriterion("user_id not between", value1, value2, "userId"); - return (Criteria) this; - } - public Criteria andStatusIsNull() { addCriterion("status is null"); return (Criteria) this; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/RetrievalDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/RetrievalDO.java new file mode 100644 index 000000000..b32291a53 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/RetrievalDO.java @@ -0,0 +1,45 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class RetrievalDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String updateUser; + private String createUser; + private String type; + private String status; + private String isDefault; + private String name; + private String chineseName; + private String schemaDesc; + private String scenariosDesc; + private String costDesc; + private String methodDesc; + private String extractorDesc; + private String retrieverDesc; + private String modulePath; + private String className; + private String method; + private String extension; + private String config; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/StatisticsDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/StatisticsDO.java new file mode 100644 index 000000000..9700b47eb --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/StatisticsDO.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class StatisticsDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String resourceTag; + private String resourceId; + private String statisticsType; + private String statisticsDate; + private Long num; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/UserModelDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/UserModelDO.java new file mode 100644 index 000000000..3afcc1fe7 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/UserModelDO.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.dataobject; + +import java.util.Date; +import lombok.Data; + +@Data +public class UserModelDO { + + private Long id; + private Date gmtCreate; + private Date gmtModified; + private String modifier; + private String creator; + private String instanceId; + private String visibility; + private String provider; + private String name; + private String config; +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/AppMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/AppMapper.java new file mode 100644 index 000000000..ba9da1872 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/AppMapper.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.api.facade.dto.common.request.AppRequest; +import com.antgroup.openspg.server.infra.dao.dataobject.AppDO; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface AppMapper { + + int insert(AppDO record); + + int deleteById(Long id); + + int update(AppDO record); + + AppDO getById(Long id); + + List selectByCondition( + @Param("request") AppRequest request, @Param("start") int start, @Param("size") int size); + + Long selectCountByCondition(@Param("request") AppRequest request); + + AppDO getByName(String name); + + List queryByCondition(AppRequest request); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ModelDetailDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ModelDetailDOMapper.java new file mode 100644 index 000000000..c3a72ed00 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ModelDetailDOMapper.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import com.antgroup.openspg.server.infra.dao.dataobject.ModelDetailDO; +import java.util.List; + +public interface ModelDetailDOMapper { + + Long insert(ModelDetailDO modelDetailDO); + + int deleteById(Long id); + + Long update(ModelDetailDO modelDetailDO); + + ModelDetailDO getById(Long id); + + List query(ModelDetailQuery modelDetailQuery); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ModelProviderDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ModelProviderDOMapper.java new file mode 100644 index 000000000..34408832f --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ModelProviderDOMapper.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.common.model.provider.ModelProviderQuery; +import com.antgroup.openspg.server.infra.dao.dataobject.ModelProviderDO; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface ModelProviderDOMapper { + + Long insert(ModelProviderDO record); + + int deleteById(Long id); + + Long update(ModelProviderDO record); + + ModelProviderDO getById(Long id); + + ModelProviderDO getByProvider(String provider); + + List query(ModelProviderQuery record); + + List selectByProviders(@Param("providerList") List providerList); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OntologyPropertyDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OntologyPropertyDOMapper.java index b6dadddf3..4fc20e140 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OntologyPropertyDOMapper.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OntologyPropertyDOMapper.java @@ -52,4 +52,6 @@ int updateByExample( int updateByPrimaryKeyWithBLOBs(OntologyPropertyDO record); int updateByPrimaryKey(OntologyPropertyDO record); + + int deleteIndexType(OntologyPropertyDO record); } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/PermissionMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/PermissionMapper.java index b719588ac..d9b9fb7ac 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/PermissionMapper.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/PermissionMapper.java @@ -80,7 +80,7 @@ public interface PermissionMapper { * @return List */ List selectByCondition( - @Param("record") PermissionDO record, @Param("start") int start, @Param("size") int size); + @Param("record") PermissionDO record, @Param("start") Long start, @Param("size") Long size); /** * This method was generated by MyBatis Generator. This method corresponds to the database table @@ -89,7 +89,7 @@ List selectByCondition( * @param record com.antgroup.openspg.server.infra.dao.dataobject.PermissionDO * @return int */ - int selectCountByCondition(PermissionDO record); + long selectCountByCondition(PermissionDO record); /** * This method was generated by MyBatis Generator. This method corresponds to the database table @@ -222,8 +222,8 @@ List selectLikeByUserNoAndRoleId( @Param("roleId") Long roleId, @Param("resourceId") Long resourceId, @Param("resourceTag") String resourceTag, - @Param("start") int start, - @Param("size") int size); + @Param("start") Long start, + @Param("size") Long size); /** * the count of selectLikeByUserNoAndRoleId @@ -234,7 +234,7 @@ List selectLikeByUserNoAndRoleId( * @param resourceTag * @return */ - Integer selectLikeCountByUserNoAndRoleId( + long selectLikeCountByUserNoAndRoleId( @Param("userNo") String userNo, @Param("roleId") Long roleId, @Param("resourceId") Long resourceId, diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProjectDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProjectDOMapper.java index b651ef748..ef1a85b7b 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProjectDOMapper.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProjectDOMapper.java @@ -42,6 +42,8 @@ public interface ProjectDOMapper { void deleteFromKgBuilderJob(Long id); + void deleteFromKgResourcePermission(Long projectId); + int insert(ProjectDO record); int insertSelective(ProjectDO record); diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProviderParamDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProviderParamDOMapper.java new file mode 100644 index 000000000..fbe82a318 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/ProviderParamDOMapper.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.common.model.providerparam.ProviderParamQuery; +import com.antgroup.openspg.server.infra.dao.dataobject.ProviderParamDO; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface ProviderParamDOMapper { + + Long insert(ProviderParamDO record); + + int deleteById(Long id); + + Long update(ProviderParamDO record); + + ProviderParamDO getById(Long id); + + List query(ProviderParamQuery record); + + ProviderParamDO getByProviderAndModelType( + @Param("provider") String provider, @Param("modelType") String modelType); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/RefDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/RefDOMapper.java new file mode 100644 index 000000000..9c79a2e34 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/RefDOMapper.java @@ -0,0 +1,48 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.infra.dao.dataobject.RefDO; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface RefDOMapper { + int deleteByPrimaryKey(Long id); + + Long insert(RefDO record); + + int insertSelective(RefDO record); + + RefDO selectByPrimaryKey(Long id); + + RefDO selectByUniqueKey(RefDO record); + + List query(RefDO record); + + int updateByPrimaryKeySelective(RefDO record); + + int updateByPrimaryKeyWithBLOBs(RefDO record); + + int updateByPrimaryKey(RefDO record); + + List getRefInfoByRightMatchRefedId(String refedId); + + int updateByUniqueKey(RefDO refDO); + + int deleteByIds(@Param("ids") List ids); + + int deleteByUniqueKey(RefDO refDO); + + int checkUniqueToken(@Param("alias") String alias, @Param("token") String token); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/RetrievalDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/RetrievalDOMapper.java new file mode 100644 index 000000000..656d58645 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/RetrievalDOMapper.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.common.model.retrieval.RetrievalQuery; +import com.antgroup.openspg.server.infra.dao.dataobject.RetrievalDO; +import java.util.List; + +public interface RetrievalDOMapper { + + Long insert(RetrievalDO record); + + int deleteById(Long id); + + Long update(RetrievalDO record); + + RetrievalDO getById(Long id); + + RetrievalDO getByName(String name); + + List query(RetrievalQuery record); + + int selectCountByQuery(RetrievalQuery record); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/UserModelDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/UserModelDOMapper.java new file mode 100644 index 000000000..be54beab0 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/UserModelDOMapper.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.mapper; + +import com.antgroup.openspg.server.common.model.usermodel.UserModelQuery; +import com.antgroup.openspg.server.infra.dao.dataobject.UserModelDO; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface UserModelDOMapper { + + Long insert(UserModelDO record); + + int deleteById(Long id); + + int deleteByIds(@Param("ids") List ids); + + Long update(UserModelDO record); + + UserModelDO getById(Long id); + + UserModelDO getByInstanceId(String instanceId); + + List query(UserModelQuery record); + + List getByProviderAndName( + @Param("provider") String provider, @Param("name") String name); + + List selectUserPrivateOrPublic(String userNo); + + Long updateBaseInfoByIds( + @Param("ids") List ids, + @Param("name") String name, + @Param("visibility") String visibility, + @Param("userNo") String userNo, + @Param("config") String config); +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/AppRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/AppRepositoryImpl.java new file mode 100644 index 000000000..70b826d25 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/AppRepositoryImpl.java @@ -0,0 +1,92 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.antgroup.openspg.server.api.facade.dto.common.request.AppRequest; +import com.antgroup.openspg.server.common.model.app.App; +import com.antgroup.openspg.server.common.service.app.AppRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.AppDO; +import com.antgroup.openspg.server.infra.dao.mapper.AppMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.AppConvertor; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class AppRepositoryImpl implements AppRepository { + + @Autowired private AppMapper appMapper; + + @Override + public Integer save(App app) { + if (app == null) { + return 0; + } + AppDO appDO = AppConvertor.toDO(app); + int insert = appMapper.insert(appDO); + app.setId(appDO.getId()); + return insert; + } + + @Override + public Integer update(App app) { + if (app == null) { + return 0; + } + return appMapper.update(AppConvertor.toDO(app)); + } + + @Override + public App queryById(Long id) { + if (id == null) { + return null; + } + return AppConvertor.toModel(appMapper.getById(id)); + } + + @Override + public List queryPage(AppRequest request, int start, int size) { + int startIndex = (Math.max(start, 1) - 1) * size; + return AppConvertor.toModelList(appMapper.selectByCondition(request, startIndex, size)); + } + + @Override + public Long selectCountByCondition(AppRequest request) { + return appMapper.selectCountByCondition(request); + } + + @Override + public Integer deleteById(Long id) { + if (id == null) { + return 0; + } + return appMapper.deleteById(id); + } + + @Override + public App queryByName(String name) { + return AppConvertor.toModel(appMapper.getByName(name)); + } + + @Override + public List queryByCondition(AppRequest request) { + List appDOList = appMapper.queryByCondition(request); + if (CollectionUtils.isEmpty(appDOList)) { + return Collections.emptyList(); + } + return appDOList.stream().map(AppConvertor::toModel).collect(Collectors.toList()); + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ModelDetailRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ModelDetailRepositoryImpl.java new file mode 100644 index 000000000..e5cfcf5b3 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ModelDetailRepositoryImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetail; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailQuery; +import com.antgroup.openspg.server.common.service.modeldetail.ModelDetailRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.ModelDetailDO; +import com.antgroup.openspg.server.infra.dao.mapper.ModelDetailDOMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.ModelDetailConvertor; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class ModelDetailRepositoryImpl implements ModelDetailRepository { + + @Autowired private ModelDetailDOMapper modelDetailDOMapper; + + @Override + public Long insert(ModelDetail record) { + ModelDetailDO modelDetailDO = ModelDetailConvertor.toDO(record); + modelDetailDOMapper.insert(modelDetailDO); + record.setId(modelDetailDO.getId()); + return record.getId(); + } + + @Override + public int deleteById(Long id) { + return modelDetailDOMapper.deleteById(id); + } + + @Override + public Long update(ModelDetail record) { + return modelDetailDOMapper.update(ModelDetailConvertor.toDO(record)); + } + + @Override + public ModelDetail getById(Long id) { + return ModelDetailConvertor.toModel(modelDetailDOMapper.getById(id)); + } + + @Override + public List query(ModelDetailQuery record) { + return ModelDetailConvertor.toModelList(modelDetailDOMapper.query(record)); + } + + @Override + public List queryDTO(ModelDetailQuery record) { + return ModelDetailConvertor.toDTOList(modelDetailDOMapper.query(record)); + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ModelProviderRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ModelProviderRepositoryImpl.java new file mode 100644 index 000000000..fe8f9ba26 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ModelProviderRepositoryImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import com.antgroup.openspg.server.common.model.provider.ModelProviderQuery; +import com.antgroup.openspg.server.common.service.provider.ModelProviderRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.ModelProviderDO; +import com.antgroup.openspg.server.infra.dao.mapper.ModelProviderDOMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.ModelProviderConvertor; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class ModelProviderRepositoryImpl implements ModelProviderRepository { + + @Autowired private ModelProviderDOMapper modelProviderDOMapper; + + @Override + public Long insert(ModelProvider record) { + ModelProviderDO modelProviderDO = ModelProviderConvertor.toDO(record); + modelProviderDOMapper.insert(modelProviderDO); + record.setId(modelProviderDO.getId()); + return record.getId(); + } + + @Override + public int deleteById(Long id) { + return modelProviderDOMapper.deleteById(id); + } + + @Override + public Long update(ModelProvider record) { + return modelProviderDOMapper.update(ModelProviderConvertor.toDO(record)); + } + + @Override + public ModelProvider getById(Long id) { + return ModelProviderConvertor.toModel(modelProviderDOMapper.getById(id)); + } + + @Override + public List query(ModelProviderQuery record) { + return ModelProviderConvertor.toModelList(modelProviderDOMapper.query(record)); + } + + @Override + public ModelProvider getByProvider(String provider) { + return ModelProviderConvertor.toModel(modelProviderDOMapper.getByProvider(provider)); + } + + @Override + public List selectByProviders(List providerList) { + return ModelProviderConvertor.toModelList( + modelProviderDOMapper.selectByProviders(providerList)); + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/PermissionRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/PermissionRepositoryImpl.java index 4f599ce3f..b04f4b1c5 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/PermissionRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/PermissionRepositoryImpl.java @@ -15,7 +15,6 @@ import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.common.util.enums.PermissionEnum; -import com.antgroup.openspg.server.api.facade.Paged; import com.antgroup.openspg.server.common.model.permission.Permission; import com.antgroup.openspg.server.common.service.permission.PermissionRepository; import com.antgroup.openspg.server.infra.dao.dataobject.PermissionDO; @@ -60,93 +59,13 @@ public Integer update(Permission permission) { } } - @Override - public List query( - Long resourceId, String resourceTag, Integer page, Integer pageSize) { - return query(null, null, resourceId, resourceTag, page, pageSize); - } - @Override public List queryByUserNoAndRoleId( - String userNo, - Long roleId, - Long resourceId, - String resourceTag, - Integer page, - Integer pageSize) { + String userNo, Long roleId, Long resourceId, String resourceTag, Long page, Long pageSize) { if (StringUtils.isBlank(userNo)) { return Lists.newArrayList(); } - return query(userNo, roleId, resourceId, resourceTag, page, pageSize); - } - - private List query( - String userNo, - Long roleId, - Long resourceId, - String resourceTag, - Integer page, - Integer pageSize) { - if (null == resourceId || StringUtils.isBlank(resourceTag)) { - return Lists.newArrayList(); - } - if (null == pageSize) { - pageSize = 10; - } - page = page > 0 ? page : 1; - int start = (page - 1) * pageSize; - List permissionList; - if (StringUtils.isBlank(userNo)) { - PermissionDO permissionDO = new PermissionDO(); - permissionDO.setResourceId(resourceId); - permissionDO.setResourceTag(resourceTag); - permissionList = permissionMapper.selectByCondition(permissionDO, start, pageSize); - } else { - permissionList = - permissionMapper.selectLikeByUserNoAndRoleId( - userNo, roleId, resourceId, resourceTag, start, pageSize); - } - if (CollectionUtils.isNotEmpty(permissionList)) { - return permissionList.stream().map(PermissionConvertor::toModel).collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - @Override - public Paged queryPage( - String userNo, Long roleId, Long resourceId, String resourceTag, Integer page, Integer size) { - Paged result = new Paged<>(); - result.setPageIdx(page); - result.setPageSize(size); - List permissionList; - PermissionDO permissionDO = new PermissionDO(); - permissionDO.setResourceTag(resourceTag); - permissionDO.setResourceId(resourceId); - permissionDO.setRoleId(roleId); - if (null == size) { - size = 10; - } - page = page > 0 ? page : 1; - int start = (page - 1) * size; - if (StringUtils.isNotBlank(userNo)) { - permissionDO.setUserNo(userNo); - result.setTotal( - (long) - permissionMapper.selectLikeCountByUserNoAndRoleId( - userNo, roleId, resourceId, resourceTag)); - permissionList = - permissionMapper.selectLikeByUserNoAndRoleId( - userNo, roleId, resourceId, resourceTag, start, size); - } else { - result.setTotal((long) permissionMapper.selectCountByCondition(permissionDO)); - permissionList = permissionMapper.selectByCondition(permissionDO, start, size); - } - - if (CollectionUtils.isNotEmpty(permissionList)) { - result.setResults( - permissionList.stream().map(PermissionConvertor::toModel).collect(Collectors.toList())); - } - return result; + return selectLikeByUserNoAndRoleId(userNo, roleId, resourceId, resourceTag, page, pageSize); } @Override @@ -209,4 +128,56 @@ public Permission selectByPrimaryKey(Long id) { PermissionDO permissionDO = permissionMapper.selectByPrimaryKey(id); return PermissionConvertor.toModel(permissionDO); } + + @Override + public List selectLikeByUserNoAndRoleId( + String userNo, Long roleId, Long resourceId, String resourceTag, Long page, Long size) { + if (null == resourceId || StringUtils.isBlank(resourceTag)) { + return Lists.newArrayList(); + } + List permissionList; + if (null == size) { + size = 10L; + } + page = page > 0 ? page : 1; + Long start = (page - 1) * size; + if (StringUtils.isNotBlank(userNo)) { + permissionList = + permissionMapper.selectLikeByUserNoAndRoleId( + userNo, roleId, resourceId, resourceTag, start, size); + } else { + PermissionDO permissionDO = new PermissionDO(); + permissionDO.setResourceTag(resourceTag); + permissionDO.setResourceId(resourceId); + permissionDO.setRoleId(roleId); + permissionList = permissionMapper.selectByCondition(permissionDO, start, size); + } + return PermissionConvertor.toModelList(permissionList); + } + + @Override + public long selectLikeCountByUserNoAndRoleId( + String userNo, Long roleId, Long resourceId, String resourceTag) { + if (null == resourceId || StringUtils.isBlank(resourceTag)) { + return 0; + } + Long count; + if (StringUtils.isNotBlank(userNo)) { + count = + permissionMapper.selectLikeCountByUserNoAndRoleId( + userNo, roleId, resourceId, resourceTag); + } else { + PermissionDO permissionDO = new PermissionDO(); + permissionDO.setResourceTag(resourceTag); + permissionDO.setResourceId(resourceId); + permissionDO.setRoleId(roleId); + count = permissionMapper.selectCountByCondition(permissionDO); + } + return count; + } + + @Override + public int deleteByResourceId(Long resourceId, String resourceTag) { + return permissionMapper.deleteByResourceId(resourceId, resourceTag); + } } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProjectRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProjectRepositoryImpl.java index 96fbd6a54..81875669d 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProjectRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProjectRepositoryImpl.java @@ -14,7 +14,8 @@ package com.antgroup.openspg.server.infra.dao.repository.common; import com.antgroup.openspg.common.util.CollectionsUtils; -import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.common.util.enums.VisibilityEnum; import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest; import com.antgroup.openspg.server.common.model.exception.ProjectException; import com.antgroup.openspg.server.common.model.project.Project; @@ -24,7 +25,6 @@ import com.antgroup.openspg.server.infra.dao.mapper.ProjectDOMapper; import com.antgroup.openspg.server.infra.dao.repository.common.convertor.ProjectConvertor; import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; -import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.stream.Collectors; @@ -53,6 +53,9 @@ public Long save(Project project) { ProjectDO projectDO = ProjectConvertor.toDO(project); projectDO.setGmtModified(new Date()); projectDO.setGmtCreate(new Date()); + if (StringUtils.isBlank(projectDO.getVisibility())) { + projectDO.setVisibility(VisibilityEnum.PRIVATE.name()); + } projectDOMapper.insert(projectDO); return projectDO.getId(); } @@ -82,6 +85,7 @@ public Integer deleteById(Long projectId) { projectDOMapper.deleteFromKgReasonTask(projectId); projectDOMapper.deleteFromKgReasonTutorial(projectId); projectDOMapper.deleteFromKgBuilderJob(projectId); + projectDOMapper.deleteFromKgResourcePermission(projectId); return projectDOMapper.deleteByPrimaryKey(projectId); } @@ -102,28 +106,26 @@ public List query(ProjectQueryRequest request) { if (request.getNamespace() != null) { criteria.andNamespaceEqualTo(request.getNamespace()); } + if (CollectionUtils.isNotEmpty(request.getProjectIdList())) { + criteria.andIdIn(request.getProjectIdList()); + } List projectDOS = projectDOMapper.selectByExample(example); return CollectionsUtils.listMap(projectDOS, ProjectConvertor::toModel); } @Override - public Paged queryPaged(ProjectQueryRequest query, int start, int size) { - Paged result = new Paged<>(); - result.setPageIdx(start); - result.setPageSize(size); - long count = projectDOMapper.selectCountByCondition(query, query.getOrderByGmtCreateDesc()); - result.setTotal(count); - List list = new ArrayList<>(); - start = start > 0 ? start : 1; - int startPage = (start - 1) * size; - List projectDOS = - projectDOMapper.selectByCondition(query, query.getOrderByGmtCreateDesc(), startPage, size); - if (CollectionUtils.isNotEmpty(projectDOS)) { - list = projectDOS.stream().map(ProjectConvertor::toModel).collect(Collectors.toList()); - } - result.setResults(list); - return result; + public List queryPageData(ProjectQueryRequest query, int start, int size) { + int startIndex = (Math.max(start, 1) - 1) * size; + return projectDOMapper + .selectByCondition(query, query.getOrderByGmtCreateDesc(), startIndex, size).stream() + .map(ProjectConvertor::toModel) + .collect(Collectors.toList()); + } + + @Override + public Long queryPageCount(ProjectQueryRequest query) { + return projectDOMapper.selectCountByCondition(query, query.getOrderByGmtCreateDesc()); } @Override diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProviderParamRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProviderParamRepositoryImpl.java new file mode 100644 index 000000000..7312a726c --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/ProviderParamRepositoryImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.antgroup.openspg.server.common.model.providerparam.ProviderParam; +import com.antgroup.openspg.server.common.model.providerparam.ProviderParamQuery; +import com.antgroup.openspg.server.common.service.providerparam.ProviderParamRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.ProviderParamDO; +import com.antgroup.openspg.server.infra.dao.mapper.ProviderParamDOMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.ProviderParamConvertor; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class ProviderParamRepositoryImpl implements ProviderParamRepository { + + @Autowired private ProviderParamDOMapper paramDOMapper; + + @Override + public Long insert(ProviderParam record) { + ProviderParamDO providerParamDO = ProviderParamConvertor.toDO(record); + paramDOMapper.insert(providerParamDO); + record.setId(providerParamDO.getId()); + return record.getId(); + } + + @Override + public int deleteById(Long id) { + return paramDOMapper.deleteById(id); + } + + @Override + public Long update(ProviderParam record) { + return paramDOMapper.update(ProviderParamConvertor.toDO(record)); + } + + @Override + public ProviderParam getById(Long id) { + return ProviderParamConvertor.toModel(paramDOMapper.getById(id)); + } + + @Override + public List query(ProviderParamQuery record) { + return ProviderParamConvertor.toModelList(paramDOMapper.query(record)); + } + + @Override + public ProviderParam getByProviderAndModelType(String provider, String modelType) { + return ProviderParamConvertor.toModel( + paramDOMapper.getByProviderAndModelType(provider, modelType)); + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/RefRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/RefRepositoryImpl.java new file mode 100644 index 000000000..1a815f70b --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/RefRepositoryImpl.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.common.model.ref.RefTypeEnum; +import com.antgroup.openspg.server.common.model.ref.RefedTypeEnum; +import com.antgroup.openspg.server.common.service.ref.RefRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.RefDO; +import com.antgroup.openspg.server.infra.dao.mapper.RefDOMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.RefConvertor; +import java.util.Date; +import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class RefRepositoryImpl implements RefRepository { + + @Autowired private RefDOMapper refDOMapper; + + @Override + public Long insert(RefInfo refInfo) { + RefDO refDO = RefConvertor.toDO(refInfo); + refDO.setGmtCreate(new Date()); + refDO.setGmtModified(new Date()); + refDOMapper.insert(refDO); + refInfo.setId(refDO.getId()); + return refInfo.getId(); + } + + @Override + public int deleteById(Long id) { + return refDOMapper.deleteByPrimaryKey(id); + } + + @Override + public int deleteByIds(List ids) { + return refDOMapper.deleteByIds(ids); + } + + @Override + public int update(RefInfo refInfo) { + return refDOMapper.updateByPrimaryKeySelective(RefConvertor.toDO(refInfo)); + } + + @Override + public RefInfo getById(Long id) { + RefDO refDO = refDOMapper.selectByPrimaryKey(id); + return RefConvertor.toModel(refDO); + } + + @Override + public RefInfo selectByUniqueKey(String refId, String refType, String refedId, String refedType) { + RefDO refDO = new RefDO(); + refDO.setRefId(refId); + refDO.setRefType(refType); + refDO.setRefedId(refedId); + refDO.setRefedType(refedType); + RefDO result = refDOMapper.selectByUniqueKey(refDO); + return RefConvertor.toModel(result); + } + + @Override + public int updateByPrimaryKeySelective(RefInfo refInfo) { + return refDOMapper.updateByPrimaryKeySelective(RefConvertor.toDO(refInfo)); + } + + @Override + public int updateByUniqueKey(RefInfo refInfo) { + return refDOMapper.updateByUniqueKey(RefConvertor.toDO(refInfo)); + } + + @Override + public List getRefInfoByRef(String refId, RefTypeEnum refType) { + RefDO refDO = new RefDO(); + List result; + refDO.setRefId(refId); + refDO.setRefType(refType.name()); + result = refDOMapper.query(refDO); + return RefConvertor.toModelList(result); + } + + @Override + public List getRefInfoByRefed(String refedId, RefedTypeEnum refedType) { + RefDO refDO = new RefDO(); + List result; + refDO.setRefedId(refedId); + refDO.setRefedType(refedType.name()); + result = refDOMapper.query(refDO); + return RefConvertor.toModelList(result); + } + + @Override + public List getRefInfoByRightMatchRefedId(String refedId) { + return RefConvertor.toModelList(refDOMapper.getRefInfoByRightMatchRefedId(refedId)); + } + + @Override + public int deleteByUniqueKey(RefInfo refInfo) { + return refDOMapper.deleteByUniqueKey(RefConvertor.toDO(refInfo)); + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/RetrievalRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/RetrievalRepositoryImpl.java new file mode 100644 index 000000000..d47848e45 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/RetrievalRepositoryImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.antgroup.openspg.common.util.CommonUtils; +import com.antgroup.openspg.server.api.facade.Paged; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.common.model.retrieval.RetrievalQuery; +import com.antgroup.openspg.server.common.service.retrieval.RetrievalRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.RetrievalDO; +import com.antgroup.openspg.server.infra.dao.mapper.RetrievalDOMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.RetrievalConvertor; +import com.google.common.collect.Lists; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class RetrievalRepositoryImpl implements RetrievalRepository { + + @Autowired private RetrievalDOMapper retrievalDOMapper; + + @Override + public Long insert(Retrieval record) { + RetrievalDO retrievalDO = RetrievalConvertor.toDO(record); + retrievalDOMapper.insert(retrievalDO); + record.setId(retrievalDO.getId()); + return retrievalDO.getId(); + } + + @Override + public int deleteById(Long id) { + return retrievalDOMapper.deleteById(id); + } + + @Override + public Long update(Retrieval record) { + return retrievalDOMapper.update(RetrievalConvertor.toDO(record)); + } + + @Override + public Retrieval getById(Long id) { + return RetrievalConvertor.toModel(retrievalDOMapper.getById(id)); + } + + @Override + public Retrieval getByName(String name) { + return RetrievalConvertor.toModel(retrievalDOMapper.getByName(name)); + } + + @Override + public Paged query(RetrievalQuery record) { + Paged pageData = new Paged(record.getPageSize(), record.getPageNo()); + int count = retrievalDOMapper.selectCountByQuery(record); + pageData.setTotal(Long.valueOf(count)); + if (count <= 0) { + pageData.setResults(Lists.newArrayList()); + return pageData; + } + CommonUtils.checkQueryPage(count, record.getPageNo(), record.getPageSize()); + pageData.setResults(RetrievalConvertor.toModelList(retrievalDOMapper.query(record))); + return pageData; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/UserModelRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/UserModelRepositoryImpl.java new file mode 100644 index 000000000..e55ae3e60 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/UserModelRepositoryImpl.java @@ -0,0 +1,129 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.constants.SpgAppConstant; +import com.antgroup.openspg.common.util.JsonUtils; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import com.antgroup.openspg.server.common.model.usermodel.UserModel; +import com.antgroup.openspg.server.common.model.usermodel.UserModelQuery; +import com.antgroup.openspg.server.common.service.provider.ModelProviderRepository; +import com.antgroup.openspg.server.common.service.usermodel.UserModelRepository; +import com.antgroup.openspg.server.infra.dao.dataobject.UserModelDO; +import com.antgroup.openspg.server.infra.dao.mapper.UserModelDOMapper; +import com.antgroup.openspg.server.infra.dao.repository.common.convertor.UserModelConvertor; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +@Repository +public class UserModelRepositoryImpl implements UserModelRepository { + + @Autowired private UserModelDOMapper userModelDOMapper; + + @Autowired private ModelProviderRepository modelProviderRepository; + + @Override + public Long insert(UserModel record) { + UserModelDO userModelDO = UserModelConvertor.toDO(record); + userModelDOMapper.insert(userModelDO); + record.setId(userModelDO.getId()); + return record.getId(); + } + + @Override + public int deleteById(Long id) { + return userModelDOMapper.deleteById(id); + } + + @Override + public int deleteByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return 0; + } + return userModelDOMapper.deleteByIds(ids); + } + + @Override + public Long update(UserModel record) { + return userModelDOMapper.update(UserModelConvertor.toDO(record)); + } + + @Override + public UserModel getById(Long id) { + return UserModelConvertor.toModel(userModelDOMapper.getById(id)); + } + + @Override + public UserModel getByInstanceId(String instanceId) { + return UserModelConvertor.toModel(userModelDOMapper.getByInstanceId(instanceId)); + } + + @Override + public List query(UserModelQuery record) { + return UserModelConvertor.toModelList(userModelDOMapper.query(record)); + } + + @Override + public List getByProviderAndName(String provider, String name) { + return UserModelConvertor.toModelList(userModelDOMapper.getByProviderAndName(provider, name)); + } + + @Override + public List selectUserPrivateOrPublic(String userNo) { + return UserModelConvertor.toModelList(userModelDOMapper.selectUserPrivateOrPublic(userNo)); + } + + @Override + public Long updateBaseInfoByIds( + List ids, String name, String visibility, String userNo, String config) { + if (CollectionUtils.isEmpty(ids)) { + return 0L; + } + return userModelDOMapper.updateBaseInfoByIds(ids, name, visibility, userNo, config); + } + + @Override + public JSONObject getByModelId(String modelId) { + if (StringUtils.isBlank(modelId)) { + return null; + } + String[] split = modelId.split("@"); + if (split.length == 2) { + String instanceId = split[0]; + UserModel userModel = getByInstanceId(instanceId); + if (userModel == null) { + return null; + } + ModelProvider provider = modelProviderRepository.getByProvider(userModel.getProvider()); + if (provider == null) { + return null; + } + JSONArray modelList = userModel.getModelList(); + for (Object o : modelList) { + JSONObject modelJson = (JSONObject) o; + if (StringUtils.equals(modelId, modelJson.getString(SpgAppConstant.MODEL_ID))) { + modelJson.put(SpgAppConstant.LOGO, provider.getLogo()); + modelJson.put(SpgAppConstant.PROVIDER, provider.getProvider()); + modelJson = JsonUtils.flatten(modelJson); + return modelJson; + } + } + } + return null; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/AppConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/AppConvertor.java new file mode 100644 index 000000000..4ec23d5b5 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/AppConvertor.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.app.App; +import com.antgroup.openspg.server.infra.dao.dataobject.AppDO; +import com.google.common.collect.Lists; +import java.util.List; + +public class AppConvertor { + + public static AppDO toDO(App app) { + if (app == null) { + return null; + } + AppDO appDO = new AppDO(); + appDO.setId(app.getId()); + appDO.setModifier(app.getUserNo()); + appDO.setCreator(app.getUserNo()); + appDO.setName(app.getName()); + appDO.setLogo(app.getLogo()); + appDO.setDescription(app.getDescription()); + if (app.getConfig() != null) { + String configStr = JSON.toJSONString(app.getConfig()); + appDO.setConfig(configStr); + } + appDO.setAlias(app.getAlias()); + return appDO; + } + + public static App toModel(AppDO appDO) { + if (appDO == null) { + return null; + } + App app = new App(); + app.setId(appDO.getId()); + app.setUserNo(appDO.getCreator()); + app.setName(appDO.getName()); + app.setLogo(appDO.getLogo()); + app.setDescription(appDO.getDescription()); + if (StringUtils.isNotBlank(appDO.getConfig())) { + JSONObject parseArray = JSON.parseObject(appDO.getConfig()); + app.setConfig(parseArray); + } + app.setAlias(appDO.getAlias()); + return app; + } + + public static List toDoList(List modelDetails) { + if (modelDetails == null) { + return null; + } + List dos = Lists.newArrayList(); + for (App provider : modelDetails) { + dos.add(toDO(provider)); + } + return dos; + } + + public static List toModelList(List modelDetailDOs) { + if (modelDetailDOs == null) { + return null; + } + List details = Lists.newArrayList(); + for (AppDO providerDO : modelDetailDOs) { + details.add(toModel(providerDO)); + } + return details; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/BuilderJobConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/BuilderJobConvertor.java index 33e4a0de8..b02c72e39 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/BuilderJobConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/BuilderJobConvertor.java @@ -36,12 +36,12 @@ public static BuilderJobDO toDO(BuilderJob job) { return jobDO; } - public static BuilderJob toModel(BuilderJobDO schedulerJobDO) { - if (null == schedulerJobDO) { + public static BuilderJob toModel(BuilderJobDO builderJobDO) { + if (null == builderJobDO) { return null; } - BuilderJob job = DozerBeanMapperUtil.map(schedulerJobDO, BuilderJob.class); + BuilderJob job = DozerBeanMapperUtil.map(builderJobDO, BuilderJob.class); if (StringUtils.isBlank(job.getLifeCycle())) { job.setLifeCycle(SchedulerEnum.LifeCycle.ONCE.name()); } @@ -72,13 +72,13 @@ public static List toDoList(List jobs) { return dos; } - public static List toModelList(List schedulerJobDOs) { - if (schedulerJobDOs == null) { + public static List toModelList(List builderJobDOS) { + if (builderJobDOS == null) { return null; } List jobs = Lists.newArrayList(); - for (BuilderJobDO schedulerJobDO : schedulerJobDOs) { - jobs.add(toModel(schedulerJobDO)); + for (BuilderJobDO builderJobDO : builderJobDOS) { + jobs.add(toModel(builderJobDO)); } return jobs; } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ConfigConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ConfigConvertor.java index fdd8bc44e..35b98ffd7 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ConfigConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ConfigConvertor.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.server.infra.dao.repository.common.convertor; +import com.alibaba.fastjson.JSON; import com.antgroup.openspg.server.common.model.config.Config; import com.antgroup.openspg.server.infra.dao.dataobject.ConfigDO; @@ -22,11 +23,20 @@ public static Config toModel(ConfigDO configDO) { if (null == configDO) { return null; } + Object config = null; + String json = configDO.getConfig().trim(); // 去除头尾的空格 + if (json.startsWith("{") && json.endsWith("}")) { + config = JSON.parseObject(json); + } else if (json.startsWith("[") && json.endsWith("]")) { + config = JSON.parseArray(json); + } else { + config = json; + } return new Config( configDO.getId(), configDO.getConfigName(), configDO.getConfigId(), - configDO.getConfig(), + config, configDO.getResourceType()); } @@ -37,7 +47,7 @@ public static ConfigDO toDO(Config config) { configDO.setProjectId(config.getProjectId()); configDO.setConfigName(config.getConfigName()); configDO.setConfigId(config.getConfigId()); - configDO.setConfig(config.getConfig()); + configDO.setConfig(JSON.toJSONString(config.getConfig())); configDO.setVersion(config.getVersion()); configDO.setDescription(config.getDescription()); configDO.setResourceId(config.getResourceId()); diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/FeedbackConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/FeedbackConvertor.java new file mode 100644 index 000000000..f51e2687c --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/FeedbackConvertor.java @@ -0,0 +1,94 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.feedback.Feedback; +import com.antgroup.openspg.server.infra.dao.dataobject.FeedbackDO; +import com.google.common.collect.Lists; +import java.util.List; + +public class FeedbackConvertor { + + public static FeedbackDO toDO(Feedback feedback) { + if (feedback == null) { + return null; + } + FeedbackDO feedbackDO = new FeedbackDO(); + feedbackDO.setId(feedback.getId()); + feedbackDO.setGmtCreate(feedback.getGmtCreate()); + feedbackDO.setGmtModified(feedback.getGmtModified()); + feedbackDO.setCreator(feedback.getCreator()); + feedbackDO.setModifier(feedback.getModifier()); + feedbackDO.setModuleType(feedback.getModuleType()); + feedbackDO.setOneCategory(feedback.getOneCategory()); + feedbackDO.setTwoCategory(feedback.getTwoCategory()); + feedbackDO.setThreeCategory(feedback.getThreeCategory()); + feedbackDO.setFourCategory(feedback.getFourCategory()); + feedbackDO.setFiveCategory(feedback.getFiveCategory()); + feedbackDO.setReactionType(feedback.getReactionType()); + if (feedback.getReason() != null) { + String reason = JSON.toJSONString(feedback.getReason()); + feedbackDO.setReason(reason); + } + return feedbackDO; + } + + public static Feedback toModel(FeedbackDO feedbackDO) { + if (feedbackDO == null) { + return null; + } + Feedback feedback = new Feedback(); + feedback.setId(feedbackDO.getId()); + feedback.setGmtCreate(feedbackDO.getGmtCreate()); + feedback.setGmtModified(feedbackDO.getGmtModified()); + feedback.setCreator(feedbackDO.getCreator()); + feedback.setModifier(feedbackDO.getModifier()); + feedback.setModuleType(feedbackDO.getModuleType()); + feedback.setOneCategory(feedbackDO.getOneCategory()); + feedback.setTwoCategory(feedbackDO.getTwoCategory()); + feedback.setThreeCategory(feedbackDO.getThreeCategory()); + feedback.setFourCategory(feedbackDO.getFourCategory()); + feedback.setFiveCategory(feedbackDO.getFiveCategory()); + feedback.setReactionType(feedbackDO.getReactionType()); + if (StringUtils.isNotBlank(feedbackDO.getReason())) { + JSONObject parseObject = JSON.parseObject(feedbackDO.getReason()); + feedback.setReason(parseObject); + } + return feedback; + } + + public static List toDoList(List feedbacks) { + if (feedbacks == null) { + return null; + } + List dos = Lists.newArrayList(); + for (Feedback feedback : feedbacks) { + dos.add(toDO(feedback)); + } + return dos; + } + + public static List toModelList(List feedbackDOS) { + if (feedbackDOS == null) { + return null; + } + List jobs = Lists.newArrayList(); + for (FeedbackDO feedbackDO : feedbackDOS) { + jobs.add(toModel(feedbackDO)); + } + return jobs; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ModelDetailConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ModelDetailConvertor.java new file mode 100644 index 000000000..9267996aa --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ModelDetailConvertor.java @@ -0,0 +1,110 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetail; +import com.antgroup.openspg.server.common.model.modeldetail.ModelDetailDTO; +import com.antgroup.openspg.server.infra.dao.dataobject.ModelDetailDO; +import com.google.common.collect.Lists; +import java.util.List; + +public class ModelDetailConvertor { + + public static ModelDetailDO toDO(ModelDetail modelDetail) { + if (modelDetail == null) { + return null; + } + ModelDetailDO modelDetailDO = new ModelDetailDO(); + modelDetailDO.setId(modelDetail.getId()); + modelDetailDO.setGmtCreate(modelDetail.getGmtCreate()); + modelDetailDO.setGmtModified(modelDetail.getGmtModified()); + modelDetailDO.setModifier(modelDetail.getModifier()); + modelDetailDO.setCreator(modelDetail.getCreator()); + modelDetailDO.setProvider(modelDetail.getProvider()); + modelDetailDO.setType(modelDetail.getType()); + modelDetailDO.setName(modelDetail.getName()); + modelDetailDO.setDescription(modelDetail.getDescription()); + if (modelDetail.getParams() != null) { + String paramsStr = JSON.toJSONString(modelDetail.getParams()); + modelDetailDO.setParams(paramsStr); + } + return modelDetailDO; + } + + public static ModelDetail toModel(ModelDetailDO modelDetailDO) { + if (modelDetailDO == null) { + return null; + } + ModelDetail modelDetail = new ModelDetail(); + modelDetail.setId(modelDetailDO.getId()); + modelDetail.setGmtCreate(modelDetailDO.getGmtCreate()); + modelDetail.setGmtModified(modelDetailDO.getGmtModified()); + modelDetail.setModifier(modelDetailDO.getModifier()); + modelDetail.setCreator(modelDetailDO.getCreator()); + modelDetail.setProvider(modelDetailDO.getProvider()); + modelDetail.setType(modelDetailDO.getType()); + modelDetail.setName(modelDetailDO.getName()); + modelDetail.setDescription(modelDetailDO.getDescription()); + if (StringUtils.isNotBlank(modelDetailDO.getParams())) { + JSONObject parseArray = JSON.parseObject(modelDetailDO.getParams()); + modelDetail.setParams(parseArray); + } + return modelDetail; + } + + public static ModelDetailDTO toDTO(ModelDetailDO modelDetailDO) { + if (modelDetailDO == null) { + return null; + } + ModelDetailDTO modelDetailDTO = new ModelDetailDTO(); + modelDetailDTO.setType(modelDetailDO.getType()); + modelDetailDTO.setName(modelDetailDO.getName()); + return modelDetailDTO; + } + + public static List toDoList(List modelDetails) { + if (modelDetails == null) { + return null; + } + List dos = Lists.newArrayList(); + for (ModelDetail provider : modelDetails) { + dos.add(toDO(provider)); + } + return dos; + } + + public static List toModelList(List modelDetailDOs) { + if (modelDetailDOs == null) { + return null; + } + List details = Lists.newArrayList(); + for (ModelDetailDO providerDO : modelDetailDOs) { + details.add(toModel(providerDO)); + } + return details; + } + + public static List toDTOList(List modelDetailDOs) { + if (modelDetailDOs == null) { + return null; + } + List jobs = Lists.newArrayList(); + for (ModelDetailDO modelDetailDO : modelDetailDOs) { + jobs.add(toDTO(modelDetailDO)); + } + return jobs; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ModelProviderConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ModelProviderConvertor.java new file mode 100644 index 000000000..eaf6c4171 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ModelProviderConvertor.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.provider.ModelProvider; +import com.antgroup.openspg.server.infra.dao.dataobject.ModelProviderDO; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; + +public class ModelProviderConvertor { + + public static ModelProviderDO toDO(ModelProvider modelProvider) { + if (modelProvider == null) { + return null; + } + ModelProviderDO modelProviderDO = new ModelProviderDO(); + modelProviderDO.setId(modelProvider.getId()); + modelProviderDO.setGmtCreate(modelProvider.getGmtCreate()); + modelProviderDO.setGmtModified(modelProvider.getGmtModified()); + modelProviderDO.setCreator(modelProvider.getCreator()); + modelProviderDO.setModifier(modelProvider.getModifier()); + modelProviderDO.setName(modelProvider.getName()); + modelProviderDO.setProvider(modelProvider.getProvider()); + modelProviderDO.setPageMode(modelProvider.getPageMode()); + modelProviderDO.setLogo(modelProvider.getLogo()); + modelProviderDO.setTags(modelProvider.getTags()); + if (modelProvider.getParams() != null) { + String paramsStr = JSON.toJSONString(modelProvider.getParams()); + modelProviderDO.setParams(paramsStr); + } + if (CollectionUtils.isNotEmpty(modelProvider.getModelType())) { + String modelTypeStr = StringUtils.join(modelProvider.getModelType(), ","); + modelProviderDO.setModelType(modelTypeStr); + } + return modelProviderDO; + } + + public static ModelProvider toModel(ModelProviderDO modelProviderDO) { + if (modelProviderDO == null) { + return null; + } + ModelProvider modelProvider = new ModelProvider(); + modelProvider.setId(modelProviderDO.getId()); + modelProvider.setGmtCreate(modelProviderDO.getGmtCreate()); + modelProvider.setGmtModified(modelProviderDO.getGmtModified()); + modelProvider.setCreator(modelProviderDO.getCreator()); + modelProvider.setModifier(modelProviderDO.getModifier()); + modelProvider.setName(modelProviderDO.getName()); + modelProvider.setProvider(modelProviderDO.getProvider()); + modelProvider.setPageMode(modelProviderDO.getPageMode()); + modelProvider.setLogo(modelProviderDO.getLogo()); + modelProvider.setTags(modelProviderDO.getTags()); + if (StringUtils.isNotBlank(modelProviderDO.getParams())) { + JSONArray parseArray = JSON.parseArray(modelProviderDO.getParams()); + modelProvider.setParams(parseArray); + } + if (StringUtils.isNotBlank(modelProviderDO.getModelType())) { + List modelTypeList = Arrays.asList(modelProviderDO.getModelType().split(",")); + modelProvider.setModelType(modelTypeList); + } + return modelProvider; + } + + public static List toDoList(List providers) { + if (providers == null) { + return null; + } + List dos = Lists.newArrayList(); + for (ModelProvider provider : providers) { + dos.add(toDO(provider)); + } + return dos; + } + + public static List toModelList(List providerDOs) { + if (providerDOs == null) { + return null; + } + List jobs = Lists.newArrayList(); + for (ModelProviderDO providerDO : providerDOs) { + jobs.add(toModel(providerDO)); + } + return jobs; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/PermissionConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/PermissionConvertor.java index 6643db5ee..fb6267995 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/PermissionConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/PermissionConvertor.java @@ -15,6 +15,8 @@ import com.antgroup.openspg.server.common.model.permission.Permission; import com.antgroup.openspg.server.infra.dao.dataobject.PermissionDO; +import com.google.common.collect.Lists; +import java.util.List; public class PermissionConvertor { @@ -42,4 +44,15 @@ public static Permission toModel(PermissionDO permissionDO) { permissionDO.getResourceTag(), permissionDO.getRoleId()); } + + public static List toModelList(List paramDOs) { + if (paramDOs == null) { + return null; + } + List permissionList = Lists.newArrayList(); + for (PermissionDO permissionDO : paramDOs) { + permissionList.add(toModel(permissionDO)); + } + return permissionList; + } } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProjectConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProjectConvertor.java index 8755688b3..0c140bc30 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProjectConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProjectConvertor.java @@ -20,7 +20,6 @@ public class ProjectConvertor { public static ProjectDO toDO(Project project) { ProjectDO projectInfoDO = new ProjectDO(); - projectInfoDO.setId(project.getId()); projectInfoDO.setName(project.getName()); projectInfoDO.setDescription(project.getDescription()); @@ -28,6 +27,8 @@ public static ProjectDO toDO(Project project) { projectInfoDO.setBizDomainId(project.getTenantId()); projectInfoDO.setNamespace(project.getNamespace()); projectInfoDO.setConfig(project.getConfig()); + projectInfoDO.setVisibility(project.getVisibility()); + projectInfoDO.setTag(project.getTag()); return projectInfoDO; } @@ -35,13 +36,16 @@ public static Project toModel(ProjectDO projectInfoDO) { if (null == projectInfoDO) { return null; } - - return new Project( - projectInfoDO.getId(), - projectInfoDO.getName(), - projectInfoDO.getDescription(), - projectInfoDO.getNamespace(), - projectInfoDO.getBizDomainId(), - projectInfoDO.getConfig()); + Project project = + new Project( + projectInfoDO.getId(), + projectInfoDO.getName(), + projectInfoDO.getDescription(), + projectInfoDO.getNamespace(), + projectInfoDO.getBizDomainId(), + projectInfoDO.getConfig(), + projectInfoDO.getTag()); + project.setVisibility(projectInfoDO.getVisibility()); + return project; } } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProviderParamConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProviderParamConvertor.java new file mode 100644 index 000000000..2b9848284 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/ProviderParamConvertor.java @@ -0,0 +1,86 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.providerparam.ProviderParam; +import com.antgroup.openspg.server.infra.dao.dataobject.ProviderParamDO; +import com.google.common.collect.Lists; +import java.util.List; + +public class ProviderParamConvertor { + + public static ProviderParamDO toDO(ProviderParam providerParam) { + if (null == providerParam) { + return null; + } + ProviderParamDO providerParamDO = new ProviderParamDO(); + providerParamDO.setId(providerParam.getId()); + providerParamDO.setGmtCreate(providerParam.getGmtCreate()); + providerParamDO.setGmtModified(providerParam.getGmtModified()); + providerParamDO.setModifier(providerParam.getModifier()); + providerParamDO.setCreator(providerParam.getCreator()); + providerParamDO.setProvider(providerParam.getProvider()); + providerParamDO.setModelType(providerParam.getModelType()); + providerParamDO.setModelName(providerParam.getModelName()); + if (providerParam.getParams() != null) { + String paramsStr = JSON.toJSONString(providerParam.getParams()); + providerParamDO.setParams(paramsStr); + } + return providerParamDO; + } + + public static ProviderParam toModel(ProviderParamDO providerParamDO) { + if (null == providerParamDO) { + return null; + } + ProviderParam providerParam = new ProviderParam(); + providerParam.setId(providerParamDO.getId()); + providerParam.setGmtCreate(providerParamDO.getGmtCreate()); + providerParam.setGmtModified(providerParamDO.getGmtModified()); + providerParam.setModifier(providerParamDO.getModifier()); + providerParam.setCreator(providerParamDO.getCreator()); + providerParam.setProvider(providerParamDO.getProvider()); + providerParam.setModelType(providerParamDO.getModelType()); + providerParam.setModelName(providerParamDO.getModelName()); + if (StringUtils.isNotBlank(providerParamDO.getParams())) { + JSONArray JSONArray = JSON.parseArray(providerParamDO.getParams()); + providerParam.setParams(JSONArray); + } + return providerParam; + } + + public static List toDoList(List params) { + if (params == null) { + return null; + } + List dos = Lists.newArrayList(); + for (ProviderParam provider : params) { + dos.add(toDO(provider)); + } + return dos; + } + + public static List toModelList(List paramDOs) { + if (paramDOs == null) { + return null; + } + List jobs = Lists.newArrayList(); + for (ProviderParamDO providerDO : paramDOs) { + jobs.add(toModel(providerDO)); + } + return jobs; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/RefConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/RefConvertor.java new file mode 100644 index 000000000..b543c3c90 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/RefConvertor.java @@ -0,0 +1,64 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.antgroup.openspg.server.common.model.ref.RefInfo; +import com.antgroup.openspg.server.infra.dao.dataobject.RefDO; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.compress.utils.Lists; + +public class RefConvertor { + + public static RefDO toDO(RefInfo refInfo) { + RefDO refDO = new RefDO(); + refDO.setId(refInfo.getId()); + refDO.setGmtCreate(refInfo.getGmtCreate()); + refDO.setGmtModified(refInfo.getGmtModified()); + refDO.setName(refInfo.getName()); + refDO.setRefId(refInfo.getRefId()); + refDO.setRefedId(refInfo.getRefedId()); + refDO.setRefType(refInfo.getRefType()); + refDO.setRefedType(refInfo.getRefedType()); + refDO.setStatus(refInfo.getStatus()); + refDO.setConfig(refInfo.getConfig()); + return refDO; + } + + public static RefInfo toModel(RefDO refDO) { + if (null == refDO) { + return null; + } + RefInfo refInfo = new RefInfo(); + refInfo.setId(refDO.getId()); + refInfo.setGmtCreate(refDO.getGmtCreate()); + refInfo.setGmtModified(refDO.getGmtModified()); + refInfo.setName(refDO.getName()); + refInfo.setRefId(refDO.getRefId()); + refInfo.setRefType(refDO.getRefType()); + refInfo.setRefedId(refDO.getRefedId()); + refInfo.setRefedType(refDO.getRefedType()); + refInfo.setStatus(refDO.getStatus()); + refInfo.setConfig(refDO.getConfig()); + return refInfo; + } + + public static List toModelList(List refDOList) { + if (CollectionUtils.isEmpty(refDOList)) { + return Lists.newArrayList(); + } + return refDOList.stream().map(RefConvertor::toModel).collect(Collectors.toList()); + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/RetrievalConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/RetrievalConvertor.java new file mode 100644 index 000000000..712b984cd --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/RetrievalConvertor.java @@ -0,0 +1,61 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.antgroup.openspg.common.util.DozerBeanMapperUtil; +import com.antgroup.openspg.server.common.model.retrieval.Retrieval; +import com.antgroup.openspg.server.infra.dao.dataobject.RetrievalDO; +import com.google.common.collect.Lists; +import java.util.List; + +public class RetrievalConvertor { + + public static RetrievalDO toDO(Retrieval retrieval) { + if (null == retrieval) { + return null; + } + RetrievalDO retrievalDO = DozerBeanMapperUtil.map(retrieval, RetrievalDO.class); + return retrievalDO; + } + + public static Retrieval toModel(RetrievalDO retrievalDO) { + if (null == retrievalDO) { + return null; + } + Retrieval retrieval = DozerBeanMapperUtil.map(retrievalDO, Retrieval.class); + return retrieval; + } + + public static List toDoList(List retrievals) { + if (retrievals == null) { + return null; + } + List dos = Lists.newArrayList(); + for (Retrieval retrieval : retrievals) { + dos.add(toDO(retrieval)); + } + return dos; + } + + public static List toModelList(List retrievalDOS) { + if (retrievalDOS == null) { + return null; + } + List retrievals = Lists.newArrayList(); + for (RetrievalDO retrievalDO : retrievalDOS) { + retrievals.add(toModel(retrievalDO)); + } + return retrievals; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/StatisticsConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/StatisticsConvertor.java new file mode 100644 index 000000000..c8c54e075 --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/StatisticsConvertor.java @@ -0,0 +1,85 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.antgroup.openspg.server.common.model.statistics.Statistics; +import com.antgroup.openspg.server.infra.dao.dataobject.StatisticsDO; +import java.util.List; +import org.apache.commons.compress.utils.Lists; + +public class StatisticsConvertor { + + public static StatisticsDO toDO(Statistics statistics) { + if (statistics == null) { + return null; + } + StatisticsDO statisticsDO = new StatisticsDO(); + statisticsDO.setId(statistics.getId()); + statisticsDO.setGmtCreate(statistics.getGmtCreate()); + statisticsDO.setGmtModified(statistics.getGmtModified()); + statisticsDO.setCreator(statistics.getCreator()); + statisticsDO.setModifier(statistics.getModifier()); + statisticsDO.setResourceTag(statistics.getResourceTag()); + statisticsDO.setResourceId(statistics.getResourceId()); + statisticsDO.setStatisticsType(statistics.getStatisticsType()); + statisticsDO.setStatisticsDate(statistics.getStatisticsDate()); + statisticsDO.setNum(statistics.getNum()); + return statisticsDO; + } + + public static Statistics toModel(StatisticsDO statisticsDO) { + if (statisticsDO == null) { + return null; + } + Statistics statistics = new Statistics(); + statistics.setId(statisticsDO.getId()); + statistics.setGmtCreate(statisticsDO.getGmtCreate()); + statistics.setGmtModified(statisticsDO.getGmtModified()); + statistics.setCreator(statisticsDO.getCreator()); + statistics.setModifier(statisticsDO.getModifier()); + statistics.setResourceTag(statisticsDO.getResourceTag()); + statistics.setResourceId(statisticsDO.getResourceId()); + statistics.setStatisticsType(statisticsDO.getStatisticsType()); + statistics.setStatisticsDate(statisticsDO.getStatisticsDate()); + statistics.setNum(statisticsDO.getNum()); + return statistics; + } + + public static List toModelList(List statisticsDOs) { + if (statisticsDOs == null) { + return null; + } + List statisticsList = Lists.newArrayList(); + for (StatisticsDO statisticsDO : statisticsDOs) { + Statistics statistics = toModel(statisticsDO); + if (statistics != null) { + statisticsList.add(statistics); + } + } + return statisticsList; + } + + public static List toDOList(List statisticsList) { + if (statisticsList == null) { + return null; + } + List statisticsDOs = Lists.newArrayList(); + for (Statistics statistics : statisticsList) { + StatisticsDO statisticsDO = toDO(statistics); + if (statisticsDO != null) { + statisticsDOs.add(statisticsDO); + } + } + return statisticsDOs; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/UserModelConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/UserModelConvertor.java new file mode 100644 index 000000000..a8715b70f --- /dev/null +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/common/convertor/UserModelConvertor.java @@ -0,0 +1,85 @@ +/* + * Copyright 2023 OpenSPG Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.server.infra.dao.repository.common.convertor; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.common.util.StringUtils; +import com.antgroup.openspg.server.common.model.usermodel.UserModel; +import com.antgroup.openspg.server.infra.dao.dataobject.UserModelDO; +import com.google.common.collect.Lists; +import java.util.List; + +public class UserModelConvertor { + + public static UserModelDO toDO(UserModel userModel) { + if (null == userModel) { + return null; + } + UserModelDO userModelDO = new UserModelDO(); + userModelDO.setId(userModel.getId()); + userModelDO.setGmtCreate(userModel.getGmtCreate()); + userModelDO.setGmtModified(userModel.getGmtModified()); + userModelDO.setModifier(userModel.getModifier()); + userModelDO.setCreator(userModel.getUserNo()); + userModelDO.setProvider(userModel.getProvider()); + userModelDO.setName(userModel.getName()); + userModelDO.setInstanceId(userModel.getInstanceId()); + userModelDO.setVisibility(userModel.getVisibility()); + userModelDO.setConfig(userModel.getConfig()); + return userModelDO; + } + + public static UserModel toModel(UserModelDO userModelDO) { + if (null == userModelDO) { + return null; + } + UserModel userModel = new UserModel(); + userModel.setId(userModelDO.getId()); + userModel.setGmtCreate(userModelDO.getGmtCreate()); + userModel.setGmtModified(userModelDO.getGmtModified()); + userModel.setModifier(userModelDO.getModifier()); + userModel.setUserNo(userModelDO.getCreator()); + userModel.setProvider(userModelDO.getProvider()); + userModel.setName(userModelDO.getName()); + userModel.setInstanceId(userModelDO.getInstanceId()); + userModel.setVisibility(userModelDO.getVisibility()); + if (StringUtils.isNotBlank(userModelDO.getConfig())) { + JSONArray jsonArray = JSON.parseArray(userModelDO.getConfig()); + userModel.setModelList(jsonArray); + } + return userModel; + } + + public static List toDoList(List params) { + if (params == null) { + return null; + } + List dos = Lists.newArrayList(); + for (UserModel provider : params) { + dos.add(toDO(provider)); + } + return dos; + } + + public static List toModelList(List paramDOs) { + if (paramDOs == null) { + return null; + } + List jobs = Lists.newArrayList(); + for (UserModelDO providerDO : paramDOs) { + jobs.add(toModel(providerDO)); + } + return jobs; + } +} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java index e794d56d0..b41701708 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java @@ -55,6 +55,9 @@ public int save(SimpleProperty simpleProperty) { @Override public int update(SimpleProperty simpleProperty) { OntologyPropertyDO propertyDO = SimplePropertyConvertor.toUpdateDO(simpleProperty); + if (propertyDO.getIndexType() == null) { + ontologyPropertyDOMapper.deleteIndexType(propertyDO); + } return ontologyPropertyDOMapper.updateByPrimaryKeySelective(propertyDO); } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java index d4a5c0c89..901dfb56f 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java @@ -13,9 +13,9 @@ package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import com.antgroup.openspg.core.schema.model.SchemaConstants; import com.antgroup.openspg.core.schema.model.SchemaExtInfo; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleProperty; import com.antgroup.openspg.server.core.schema.service.type.model.SimpleSPGType; import java.util.stream.Collectors; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java index ba99eb185..ea1169fe1 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.BasicInfo; import com.antgroup.openspg.core.schema.model.OntologyId; @@ -27,7 +28,6 @@ import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; import com.antgroup.openspg.core.schema.model.type.VisibleScopeEnum; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.type.model.OperatorConfig; import com.antgroup.openspg.server.core.schema.service.type.model.ProjectOntologyRel; import com.antgroup.openspg.server.core.schema.service.type.model.SimpleSPGType; @@ -192,6 +192,7 @@ public static SimpleSPGType toSpgType( constraintItems); } case ENTITY_TYPE: + case INDEX_TYPE: case EVENT_TYPE: { return new SimpleSPGType( diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java index 3f23b282f..4dca4e807 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java @@ -13,6 +13,7 @@ package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.BasicInfo; import com.antgroup.openspg.core.schema.model.OntologyId; @@ -28,7 +29,6 @@ import com.antgroup.openspg.core.schema.model.semantic.SPGOntologyEnum; import com.antgroup.openspg.core.schema.model.type.MultiVersionConfig; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleProperty; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDO; import com.antgroup.openspg.server.infra.dao.repository.schema.enums.MapTypeEnum; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java index 3335ee745..9f514978a 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java @@ -14,6 +14,7 @@ package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.common.util.SchemaJsonUtils; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.BasicInfo; import com.antgroup.openspg.core.schema.model.OntologyId; @@ -23,7 +24,6 @@ import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier; import com.antgroup.openspg.core.schema.model.predicate.EncryptTypeEnum; import com.antgroup.openspg.core.schema.model.type.MultiVersionConfig; -import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleSubProperty; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDO; import com.antgroup.openspg.server.infra.dao.repository.schema.enums.MapTypeEnum; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java index c52052ca8..bcfbd996f 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java @@ -24,7 +24,9 @@ public enum EntityCategoryEnum { CONCEPT, - EVENT; + EVENT, + + INDEX; public static EntityCategoryEnum getEnum(String val) { for (EntityCategoryEnum entityCategoryEnum : EntityCategoryEnum.values()) { @@ -47,6 +49,8 @@ public static EntityCategoryEnum getBySchemaType(SPGTypeEnum spgTypeEnum) { return CONCEPT; case EVENT_TYPE: return EVENT; + case INDEX_TYPE: + return INDEX; default: throw new IllegalArgumentException("illegal type=" + spgTypeEnum.name()); } @@ -65,6 +69,8 @@ public static SPGTypeEnum toSpgType(String name) { return SPGTypeEnum.CONCEPT_TYPE; case EVENT: return SPGTypeEnum.EVENT_TYPE; + case INDEX: + return SPGTypeEnum.INDEX_TYPE; default: throw new IllegalArgumentException("illegal type=" + name); } diff --git a/server/infra/dao/src/main/resources/mapper/AppDOMapper.xml b/server/infra/dao/src/main/resources/mapper/AppDOMapper.xml new file mode 100644 index 000000000..7d37f64c3 --- /dev/null +++ b/server/infra/dao/src/main/resources/mapper/AppDOMapper.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, creator, modifier, name, description, logo, config, alias + + + + + SELECT LAST_INSERT_ID() + + insert into kg_app (gmt_create, gmt_modified, + creator, modifier, name, description, + logo, config, alias + ) + values (CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, + #{creator,jdbcType=VARCHAR}, #{modifier,jdbcType=VARCHAR}, #{name}, #{description,jdbcType=VARCHAR}, + #{logo,jdbcType=VARCHAR}, #{config,jdbcType=VARCHAR}, #{alias,jdbcType=VARCHAR} + ) + + + + delete from kg_app where id = #{id,jdbcType=BIGINT} + + + + UPDATE kg_app + + gmt_modified = CURRENT_TIMESTAMP, + + modifier = #{modifier,jdbcType=VARCHAR}, + + + name = #{name,jdbcType=VARCHAR}, + + + description = #{description,jdbcType=VARCHAR}, + + + logo = #{logo,jdbcType=VARCHAR}, + + + config = #{config,jdbcType=VARCHAR}, + + + alias = #{alias,jdbcType=VARCHAR} + + + where id = #{id,jdbcType=BIGINT} + + + + + + + and creator = #{request.userNo,jdbcType=VARCHAR} + + + and name like concat('%',#{request.name,jdbcType=VARCHAR},'%') + + + and id in + + #{id} + + + + + + + + + + + + + + + AND id = #{id,jdbcType=BIGINT} + + + AND name = #{name,jdbcType=VARCHAR} + + + AND logo = #{logo,jdbcType=VARCHAR} + + + AND description = #{description,jdbcType=VARCHAR} + + + AND creator = #{userNo,jdbcType=VARCHAR} + + + AND alias = #{alias,jdbcType=VARCHAR} + + + AND id IN + + #{id} + + + + + \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/BuilderJobDOMapper.xml b/server/infra/dao/src/main/resources/mapper/BuilderJobDOMapper.xml index f28edb471..c656fb9a8 100644 --- a/server/infra/dao/src/main/resources/mapper/BuilderJobDOMapper.xml +++ b/server/infra/dao/src/main/resources/mapper/BuilderJobDOMapper.xml @@ -33,10 +33,11 @@ + - id, gmt_create, gmt_modified, create_user, modify_user, project_id, task_id, job_name, chunk_num, file_url, status, type, data_source_type, version, extension, computing_conf, life_cycle, action + id, gmt_create, gmt_modified, create_user, modify_user, project_id, task_id, job_name, chunk_num, file_url, status, type, data_source_type, version, extension, computing_conf, life_cycle, action, retrievals @@ -47,13 +48,13 @@ create_user, modify_user, task_id, job_name, chunk_num, file_url, status, type, data_source_type, version, extension, - computing_conf, life_cycle, action + computing_conf, life_cycle, action, retrievals ) values (#{projectId,jdbcType=BIGINT}, #{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{createUser,jdbcType=VARCHAR}, #{modifyUser,jdbcType=VARCHAR}, #{taskId,jdbcType=BIGINT}, #{jobName,jdbcType=VARCHAR}, #{chunkNum,jdbcType=BIGINT}, #{fileUrl,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{dataSourceType,jdbcType=VARCHAR}, #{version,jdbcType=VARCHAR}, #{extension,jdbcType=LONGVARCHAR}, - #{computingConf,jdbcType=LONGVARCHAR}, #{lifeCycle,jdbcType=VARCHAR}, #{action,jdbcType=VARCHAR} + #{computingConf,jdbcType=LONGVARCHAR}, #{lifeCycle,jdbcType=VARCHAR}, #{action,jdbcType=VARCHAR}, #{retrievals,jdbcType=VARCHAR} ) @@ -107,6 +108,9 @@ action = #{action,jdbcType=VARCHAR}, + + retrievals = #{retrievals,jdbcType=VARCHAR}, + where id = #{id,jdbcType=BIGINT} @@ -169,6 +173,9 @@ and action = #{action,jdbcType=VARCHAR} + + and retrievals like concat('%', #{retrievals,jdbcType=VARCHAR}, '%') + and gmt_create >= #{startCreateTime,jdbcType=TIMESTAMP} diff --git a/server/infra/dao/src/main/resources/mapper/ModelDetailDOMapper.xml b/server/infra/dao/src/main/resources/mapper/ModelDetailDOMapper.xml new file mode 100644 index 000000000..48b02b932 --- /dev/null +++ b/server/infra/dao/src/main/resources/mapper/ModelDetailDOMapper.xml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, creator, modifier, provider, `type`, `name`, `description`, params + + + + + SELECT LAST_INSERT_ID() + + insert into kg_model_detail (gmt_create, gmt_modified, + creator, modifier, provider, + `type`, `name`, `description`, params + ) + values (CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, + #{creator,jdbcType=VARCHAR}, #{modifier,jdbcType=VARCHAR}, #{provider,jdbcType=VARCHAR}, + #{type,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR}, #{params,jdbcType=VARCHAR} + ) + + + + delete from kg_model_detail where id = #{id,jdbcType=BIGINT} + + + + UPDATE kg_model_detail + + gmt_modified = CURRENT_TIMESTAMP, + + modifier = #{modifier,jdbcType=VARCHAR}, + + + provider = #{provider,jdbcType=VARCHAR}, + + + `type` = #{type,jdbcType=VARCHAR}, + + + `name` = #{name,jdbcType=VARCHAR}, + + + `description` = #{description,jdbcType=VARCHAR}, + + + params = #{params,jdbcType=VARCHAR} + + + where id = #{id,jdbcType=BIGINT} + + + + + + + and provider = #{provider,jdbcType=VARCHAR} + + + and type = #{type,jdbcType=VARCHAR} + + + and name = #{name,jdbcType=VARCHAR} + + + + + \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/ModelProviderDOMapper.xml b/server/infra/dao/src/main/resources/mapper/ModelProviderDOMapper.xml new file mode 100644 index 000000000..813b1afe1 --- /dev/null +++ b/server/infra/dao/src/main/resources/mapper/ModelProviderDOMapper.xml @@ -0,0 +1,159 @@ + + + + + + + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, creator, modifier, name, provider, status, page_mode, model_type, logo, tags, params + + + + + SELECT LAST_INSERT_ID() + + insert into kg_model_provider (gmt_create, gmt_modified, + creator, modifier, name, + provider, status, page_mode, + model_type, logo, tags, params + ) + values (CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, + #{creator,jdbcType=VARCHAR}, #{modifier,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, + #{provider,jdbcType=VARCHAR}, #{status,jdbcType=BIGINT}, #{pageMode,jdbcType=VARCHAR}, + #{modelType,jdbcType=VARCHAR}, #{logo,jdbcType=VARCHAR}, #{tags,jdbcType=VARCHAR}, #{params,jdbcType=VARCHAR} + ) + + + + delete from kg_model_provider where id = #{id,jdbcType=BIGINT} + + + + UPDATE kg_model_provider + + gmt_modified = CURRENT_TIMESTAMP, + + modifier = #{modifier,jdbcType=VARCHAR}, + + + name = #{name,jdbcType=VARCHAR}, + + + provider = #{provider,jdbcType=VARCHAR}, + + + status = #{status,jdbcType=BIGINT}, + + + page_mode = #{pageMode,jdbcType=VARCHAR}, + + + model_type = #{modelType,jdbcType=VARCHAR} + + + logo = #{logo,jdbcType=VARCHAR} + + + tags = #{tags,jdbcType=VARCHAR} + + + params = #{params,jdbcType=VARCHAR} + + + where id = #{id,jdbcType=BIGINT} + + + + + + + + + + status = 1 + + and id = #{id,jdbcType=BIGINT} + + + and name = #{name,jdbcType=VARCHAR} + + + and provider = #{provider,jdbcType=VARCHAR} + + + and status = #{status,jdbcType=BIGINT} + + + and page_mode = #{pageMode,jdbcType=VARCHAR} + + + and FIND_IN_SET(#{modelType,jdbcType=VARCHAR}, model_type) + + + + + + \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/OntologyPropertyDOMapper.xml b/server/infra/dao/src/main/resources/mapper/OntologyPropertyDOMapper.xml index f717b71ff..0bcaefee6 100644 --- a/server/infra/dao/src/main/resources/mapper/OntologyPropertyDOMapper.xml +++ b/server/infra/dao/src/main/resources/mapper/OntologyPropertyDOMapper.xml @@ -162,6 +162,11 @@ + + update kg_ontology_entity_property_range + set index_type = null + where id = #{id,jdbcType=BIGINT} + insert into kg_ontology_entity_property_range (id, domain_id, property_name, range_id, property_name_zh, constraint_id, diff --git a/server/infra/dao/src/main/resources/mapper/PermissionMapper.xml b/server/infra/dao/src/main/resources/mapper/PermissionMapper.xml index 87542b0c2..5124c0d39 100644 --- a/server/infra/dao/src/main/resources/mapper/PermissionMapper.xml +++ b/server/infra/dao/src/main/resources/mapper/PermissionMapper.xml @@ -91,7 +91,7 @@ - + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, creator, modifier, provider, model_type, model_name, params + + + + + SELECT LAST_INSERT_ID() + + insert into kg_provider_param (gmt_create, gmt_modified, + creator, modifier, provider, + model_type, model_name, params + ) + values (CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, + #{creator,jdbcType=VARCHAR}, #{modifier,jdbcType=VARCHAR}, #{provider,jdbcType=VARCHAR}, + #{modelType,jdbcType=VARCHAR}, #{modelName,jdbcType=BIGINT}, #{params,jdbcType=VARCHAR} + ) + + + + delete from kg_provider_param where id = #{id,jdbcType=BIGINT} + + + + UPDATE kg_provider_param + + gmt_modified = CURRENT_TIMESTAMP, + + modifier = #{modifier,jdbcType=VARCHAR}, + + + provider = #{provider,jdbcType=VARCHAR}, + + + model_type = #{modelType,jdbcType=VARCHAR}, + + + model_name = #{modelName,jdbcType=VARCHAR}, + + + params = #{params,jdbcType=VARCHAR} + + + where id = #{id,jdbcType=BIGINT} + + + + + + + and id = #{id,jdbcType=BIGINT} + + + and provider = #{provider,jdbcType=VARCHAR} + + + and model_type = #{modelType,jdbcType=VARCHAR} + + + + + + + \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/RefDOMapper.xml b/server/infra/dao/src/main/resources/mapper/RefDOMapper.xml new file mode 100644 index 000000000..840dbb863 --- /dev/null +++ b/server/infra/dao/src/main/resources/mapper/RefDOMapper.xml @@ -0,0 +1,271 @@ + + + + + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, name, ref_id, ref_type, refed_id, refed_type,status + + + config + + + + + + + + + + + delete from kg_ref + where id = #{id,jdbcType=BIGINT} + + + + delete from kg_ref where id in + + #{item} + + + + + delete from kg_ref + where + ref_id = #{refId,jdbcType=VARCHAR} + and ref_type = #{refType,jdbcType=VARCHAR} + and refed_id = #{refedId,jdbcType=VARCHAR} + and refed_type = #{refedType,jdbcType=VARCHAR} + + + + + SELECT LAST_INSERT_ID() + + insert into kg_ref (gmt_create, gmt_modified, name, + ref_id, ref_type, refed_id, + refed_type, config, status) + values (#{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{name,jdbcType=VARCHAR}, + #{refId,jdbcType=VARCHAR}, #{refType,jdbcType=VARCHAR}, #{refedId,jdbcType=VARCHAR}, + #{refedType,jdbcType=VARCHAR}, #{config,jdbcType=LONGVARCHAR}, #{status,jdbcType=INTEGER}) + + + + SELECT LAST_INSERT_ID() + + insert into kg_ref + + + gmt_create, + + + gmt_modified, + + + name, + + + ref_id, + + + ref_type, + + + refed_id, + + + refed_type, + + + status, + + + config, + + + + + #{gmtCreate,jdbcType=TIMESTAMP}, + + + #{gmtModified,jdbcType=TIMESTAMP}, + + + #{name,jdbcType=VARCHAR}, + + + #{refId,jdbcType=VARCHAR}, + + + #{refType,jdbcType=VARCHAR}, + + + #{refedId,jdbcType=VARCHAR}, + + + #{refedType,jdbcType=VARCHAR}, + + + #{status,jdbcType=INTEGER}, + + + #{config,jdbcType=LONGVARCHAR}, + + + + + update kg_ref + + + gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, + + + gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, + + + name = #{name,jdbcType=VARCHAR}, + + + ref_id = #{refId,jdbcType=VARCHAR}, + + + ref_type = #{refType,jdbcType=VARCHAR}, + + + refed_id = #{refedId,jdbcType=VARCHAR}, + + + refed_type = #{refedType,jdbcType=VARCHAR}, + + + config = #{config,jdbcType=LONGVARCHAR}, + + + status = #{status,jdbcType=INTEGER}, + + + where id = #{id,jdbcType=BIGINT} + + + update kg_ref + set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, + gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, + name = #{name,jdbcType=VARCHAR}, + ref_id = #{refId,jdbcType=VARCHAR}, + ref_type = #{refType,jdbcType=VARCHAR}, + refed_id = #{refedId,jdbcType=VARCHAR}, + refed_type = #{refedType,jdbcType=VARCHAR}, + status = #{status,jdbcType=INTEGER}, + config = #{config,jdbcType=LONGVARCHAR} + where id = #{id,jdbcType=BIGINT} + + + update kg_ref + set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, + gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, + name = #{name,jdbcType=VARCHAR}, + ref_id = #{refId,jdbcType=VARCHAR}, + ref_type = #{refType,jdbcType=VARCHAR}, + refed_id = #{refedId,jdbcType=VARCHAR}, + status = #{status,jdbcType=INTEGER}, + refed_type = #{refedType,jdbcType=VARCHAR} + where id = #{id,jdbcType=BIGINT} + + + + update kg_ref + + + gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, + + + name = #{name,jdbcType=VARCHAR}, + + + config = #{config,jdbcType=LONGVARCHAR}, + + + status = #{status,jdbcType=INTEGER}, + + + where + ref_id = #{refId,jdbcType=VARCHAR} + and ref_type = #{refType,jdbcType=VARCHAR} + and refed_id = #{refedId,jdbcType=VARCHAR} + and refed_type = #{refedType,jdbcType=VARCHAR} + + + \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/ReleaseDOMapper.xml b/server/infra/dao/src/main/resources/mapper/ReleaseDOMapper.xml index a78c594f9..6a2f12481 100644 --- a/server/infra/dao/src/main/resources/mapper/ReleaseDOMapper.xml +++ b/server/infra/dao/src/main/resources/mapper/ReleaseDOMapper.xml @@ -20,7 +20,7 @@ - + @@ -90,7 +90,7 @@ - id, gmt_create, gmt_modified, project_id, version, user_id, status + id, gmt_create, gmt_modified, project_id, version, user_no, status schema_view, description, change_procedure_id, operation_detail, error_detail, operation_info @@ -144,13 +144,14 @@ - insert into kg_ontology_release (id, gmt_create, gmt_modified, - project_id, version, user_id, + insert into kg_ontology_release ( + id, gmt_create, gmt_modified, + project_id, version, user_id, user_no, status, schema_view, description, change_procedure_id, operation_detail, error_detail, operation_info) values (#{id,jdbcType=BIGINT}, #{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, - #{projectId,jdbcType=BIGINT}, #{version,jdbcType=INTEGER}, #{userId,jdbcType=VARCHAR}, + #{projectId,jdbcType=BIGINT}, #{version,jdbcType=INTEGER}, 0, #{userNo,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{schemaView,jdbcType=LONGVARCHAR}, #{description,jdbcType=LONGVARCHAR}, #{changeProcedureId,jdbcType=LONGVARCHAR}, #{operationDetail,jdbcType=LONGVARCHAR}, #{errorDetail,jdbcType=LONGVARCHAR}, #{operationInfo,jdbcType=LONGVARCHAR}) @@ -173,8 +174,9 @@ version, - - user_id, + user_id, + + user_no, status, @@ -214,8 +216,9 @@ #{version,jdbcType=INTEGER}, - - #{userId,jdbcType=VARCHAR}, + 0, + + #{userNo,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, @@ -264,8 +267,8 @@ version = #{record.version,jdbcType=INTEGER}, - - user_id = #{record.userId,jdbcType=VARCHAR}, + + user_no = #{record.userNo,jdbcType=VARCHAR}, status = #{record.status,jdbcType=VARCHAR}, @@ -300,7 +303,7 @@ gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, project_id = #{record.projectId,jdbcType=BIGINT}, version = #{record.version,jdbcType=INTEGER}, - user_id = #{record.userId,jdbcType=VARCHAR}, + user_no = #{record.userNo,jdbcType=VARCHAR}, status = #{record.status,jdbcType=VARCHAR}, schema_view = #{record.schemaView,jdbcType=LONGVARCHAR}, description = #{record.description,jdbcType=LONGVARCHAR}, @@ -319,7 +322,7 @@ gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, project_id = #{record.projectId,jdbcType=BIGINT}, version = #{record.version,jdbcType=INTEGER}, - user_id = #{record.userId,jdbcType=VARCHAR}, + user_no = #{record.userNo,jdbcType=VARCHAR}, status = #{record.status,jdbcType=VARCHAR} @@ -340,8 +343,8 @@ version = #{version,jdbcType=INTEGER}, - - user_id = #{userId,jdbcType=VARCHAR}, + + user_no = #{userNo,jdbcType=VARCHAR}, status = #{status,jdbcType=VARCHAR}, @@ -373,7 +376,7 @@ gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, project_id = #{projectId,jdbcType=BIGINT}, version = #{version,jdbcType=INTEGER}, - user_id = #{userId,jdbcType=VARCHAR}, + user_no = #{userNo,jdbcType=VARCHAR}, status = #{status,jdbcType=VARCHAR}, schema_view = #{schemaView,jdbcType=LONGVARCHAR}, description = #{description,jdbcType=LONGVARCHAR}, @@ -389,7 +392,7 @@ gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, project_id = #{projectId,jdbcType=BIGINT}, version = #{version,jdbcType=INTEGER}, - user_id = #{userId,jdbcType=VARCHAR}, + user_no = #{userNo,jdbcType=VARCHAR}, status = #{status,jdbcType=VARCHAR} where id = #{id,jdbcType=BIGINT} diff --git a/server/infra/dao/src/main/resources/mapper/RetrievalDOMapper.xml b/server/infra/dao/src/main/resources/mapper/RetrievalDOMapper.xml new file mode 100644 index 000000000..de1157583 --- /dev/null +++ b/server/infra/dao/src/main/resources/mapper/RetrievalDOMapper.xml @@ -0,0 +1,232 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, create_user, update_user, type, status, is_default, name, chinese_name, schema_desc, scenarios_desc, cost_desc, method_desc, extractor_desc, retriever_desc, module_path, class_name, method, extension, config + + + + + SELECT LAST_INSERT_ID() + + insert into kg_retrieval (gmt_create, gmt_modified, + create_user, update_user, type, status, is_default, + name, chinese_name, schema_desc, + scenarios_desc, cost_desc, method_desc, extractor_desc, retriever_desc, + module_path, class_name, method, extension, config + ) + values (#{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, + #{createUser,jdbcType=VARCHAR}, #{updateUser,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{isDefault,jdbcType=VARCHAR}, + #{name,jdbcType=VARCHAR}, #{chineseName,jdbcType=VARCHAR}, #{schemaDesc,jdbcType=LONGVARCHAR}, #{scenariosDesc,jdbcType=LONGVARCHAR}, #{costDesc,jdbcType=LONGVARCHAR}, + #{methodDesc,jdbcType=LONGVARCHAR}, #{extractorDesc,jdbcType=LONGVARCHAR}, #{retrieverDesc,jdbcType=LONGVARCHAR}, #{modulePath,jdbcType=VARCHAR}, + #{className,jdbcType=VARCHAR}, #{method,jdbcType=VARCHAR}, #{extension,jdbcType=LONGVARCHAR}, #{config,jdbcType=LONGVARCHAR} + ) + + + + delete from kg_retrieval where id = #{id,jdbcType=BIGINT} + + + + update kg_retrieval + + gmt_modified = CURRENT_TIMESTAMP, + + update_user = #{updateUser,jdbcType=VARCHAR}, + + + type = #{type,jdbcType=VARCHAR}, + + + status = #{status,jdbcType=VARCHAR}, + + + is_default = #{isDefault,jdbcType=VARCHAR}, + + + name = #{name,jdbcType=VARCHAR}, + + + chinese_name = #{chineseName,jdbcType=VARCHAR}, + + + schema_desc = #{schemaDesc,jdbcType=LONGVARCHAR}, + + + scenarios_desc = #{scenariosDesc,jdbcType=LONGVARCHAR}, + + + cost_desc = #{costDesc,jdbcType=LONGVARCHAR}, + + + method_desc = #{methodDesc,jdbcType=LONGVARCHAR}, + + + extractor_desc = #{extractorDesc,jdbcType=LONGVARCHAR}, + + + retriever_desc = #{retrieverDesc,jdbcType=LONGVARCHAR}, + + + module_path = #{modulePath,jdbcType=VARCHAR}, + + + class_name = #{className,jdbcType=VARCHAR}, + + + method = #{method,jdbcType=VARCHAR}, + + + extension = #{extension,jdbcType=LONGVARCHAR}, + + + config = #{config,jdbcType=LONGVARCHAR}, + + + where id = #{id,jdbcType=BIGINT} + + + + + + + + + and ( name like concat('%', #{keyword}, '%') or + chinese_name like concat('%', #{keyword}, '%') or + schema_desc like concat('%', #{keyword}, '%') or + scenarios_desc like concat('%', #{keyword}, '%') or + cost_desc like concat('%', #{keyword}, '%') or + method_desc like concat('%', #{keyword}, '%') or + module_path like concat('%', #{keyword}, '%') or + class_name like concat('%', #{keyword}, '%') or + method like concat('%', #{keyword}, '%') ) + + + create_user = #{createUser,jdbcType=VARCHAR} + + + and update_user = #{updateUser,jdbcType=VARCHAR} + + + and type = #{type,jdbcType=VARCHAR} + + + and status = #{status,jdbcType=VARCHAR}, + + + and is_default = #{isDefault,jdbcType=VARCHAR}, + + + and name like concat('%', #{name,jdbcType=VARCHAR}, '%') + + + and chinese_name like concat('%', #{chineseName,jdbcType=VARCHAR}, '%') + + + and schema_desc like concat('%', #{schemaDesc,jdbcType=LONGVARCHAR}, '%') + + + and scenarios_desc like concat('%', #{scenariosDesc,jdbcType=LONGVARCHAR}, '%') + + + and cost_desc like concat('%', #{costDesc,jdbcType=LONGVARCHAR}, '%') + + + and module_path = #{modulePath,jdbcType=VARCHAR} + + + and class_name = #{className,jdbcType=VARCHAR} + + + and method = #{method,jdbcType=VARCHAR} + + + and extension like concat('%', #{extension,jdbcType=LONGVARCHAR}, '%') + + + and config like concat('%', #{config,jdbcType=LONGVARCHAR}, '%') + + + and id in + + #{item} + + + + + + + + + \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/UserModelDOMapper.xml b/server/infra/dao/src/main/resources/mapper/UserModelDOMapper.xml new file mode 100644 index 000000000..033c44fb1 --- /dev/null +++ b/server/infra/dao/src/main/resources/mapper/UserModelDOMapper.xml @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + id, gmt_create, gmt_modified, creator, modifier, instance_id, visibility, provider, name, config + + + + + SELECT LAST_INSERT_ID() + + insert into kg_user_model (gmt_create, gmt_modified, + creator, modifier, instance_id, visibility, + provider, name, config + ) + values (CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, + #{creator,jdbcType=VARCHAR}, #{modifier,jdbcType=VARCHAR}, #{instanceId}, #{visibility,jdbcType=VARCHAR}, + #{provider,jdbcType=VARCHAR}, #{name,jdbcType=BIGINT}, #{config,jdbcType=VARCHAR} + ) + + + + delete from kg_user_model where id = #{id,jdbcType=BIGINT} + + + + delete from kg_user_model + where id in + + #{id,jdbcType=BIGINT} + + + + + UPDATE kg_user_model + + gmt_modified = CURRENT_TIMESTAMP, + + modifier = #{modifier,jdbcType=VARCHAR}, + + + visibility = #{visibility,jdbcType=VARCHAR}, + + + provider = #{provider,jdbcType=VARCHAR}, + + + name = #{name,jdbcType=VARCHAR}, + + + config = #{config,jdbcType=VARCHAR} + + + where id = #{id,jdbcType=BIGINT} + + + + + + + + + and creator = #{userNo,jdbcType=BIGINT} + + + and provider = #{provider,jdbcType=VARCHAR} + + + and visibility = #{visibility,jdbcType=VARCHAR} + + + and name = #{name,jdbcType=VARCHAR} + + + + + + + + + + + UPDATE kg_user_model + + gmt_modified = CURRENT_TIMESTAMP, + + modifier = #{userNo,jdbcType=VARCHAR}, + + + visibility = #{visibility,jdbcType=VARCHAR}, + + + name = #{name,jdbcType=VARCHAR}, + + + config = #{config,jdbcType=VARCHAR} + + + where id in + + #{id,jdbcType=BIGINT} + + + \ No newline at end of file